def test_replicas_max_abosulute_delay(): stop_replica_pod, stop_replica_svc, insert_pod, insert_svc = random_pod_choice_for_callbacks() create_replicated_table_on_cluster() prometheus_scrape_interval = 30 def restart_clickhouse_and_insert_to_replicated_table(): with When(f"stop replica fetches on {stop_replica_svc}"): sql = "SYSTEM STOP FETCHES default.test_repl" kubectl.kubectl( f"exec -n {kubectl.namespace} {stop_replica_pod} -c clickhouse -- clickhouse-client -q \"{sql}\"", ok_to_fail=True, ) sql = "INSERT INTO default.test_repl SELECT now(), number FROM numbers(100000)" kubectl.kubectl( f"exec -n {kubectl.namespace} {insert_pod} -c clickhouse -- clickhouse-client -q \"{sql}\"", ) with Then("check ClickHouseReplicasMaxAbsoluteDelay firing"): fired = wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", True, labels={"hostname": stop_replica_svc}, time_range='60s', sleep_time=prometheus_scrape_interval*2, callback=restart_clickhouse_and_insert_to_replicated_table) assert fired, error("can't get ClickHouseReadonlyReplica alert in firing state") clickhouse.clickhouse_query( chi["metadata"]["name"], "SYSTEM START FETCHES; SYSTEM RESTART REPLICAS; SYSTEM SYNC REPLICA default.test_repl", timeout=240 ) with Then("check ClickHouseReplicasMaxAbsoluteDelay gone away"): resolved = wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", False, labels={"hostname": stop_replica_svc}) assert resolved, error("can't check ClickHouseReplicasMaxAbsoluteDelay alert is gone away") drop_replicated_table_on_cluster()
def run_queries_with_priority(): sql = "" for i in range(50): sql += f"SET priority={i % 20};SELECT uniq(number) FROM numbers(20000000):" cmd = f"echo \\\"{sql} SELECT 1\\\" | xargs -i'{{}}' --no-run-if-empty -d ':' -P 20 clickhouse-client --time -m -n -q \\\"{{}}\\\"" kubectl.kubectl(f"exec {priority_pod} -- bash -c \"{cmd}\"", timeout=120) clickhouse.clickhouse_query( chi["metadata"]["name"], "SELECT event_time, CurrentMetric_QueryPreempted FROM system.metric_log WHERE CurrentMetric_QueryPreempted > 0", host=priority_svc, )
def test_longest_running_query(): long_running_pod, long_running_svc, _, _ = random_pod_choice_for_callbacks() # 600s trigger + 2*30s - double prometheus scraping interval clickhouse.clickhouse_query(chi["metadata"]["name"], "SELECT now(),sleepEachRow(1),number FROM system.numbers LIMIT 660", host=long_running_svc, timeout=670) with Then("check ClickHouseLongestRunningQuery firing"): fired = wait_alert_state("ClickHouseLongestRunningQuery", "firing", True, labels={"hostname": long_running_svc}, time_range='30s', sleep_time=5) assert fired, error("can't get ClickHouseLongestRunningQuery alert in firing state") with Then("check ClickHouseLongestRunningQuery gone away"): resolved = wait_alert_state("ClickHouseLongestRunningQuery", "firing", False, labels={"hostname": long_running_svc}) assert resolved, error("can't check ClickHouseLongestRunningQuery alert is gone away")
def reboot_clickhouse_and_distributed_exection(): # we need 70 delayed files for catch insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 10000' select_sql = 'SELECT count() FROM default.test_distr' with Then("reboot clickhouse-server pod"): kubectl.kubectl( f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1", ok_to_fail=True, ) with And("Insert to distributed table"): clickhouse.clickhouse_query(chi["metadata"]["name"], insert_sql, host=delayed_pod, ns=kubectl.namespace) with And("Select from distributed table"): clickhouse.clickhouse_query_with_error(chi["metadata"]["name"], select_sql, host=delayed_pod, ns=kubectl.namespace)
def insert_many_parts_to_clickhouse(): stop_merges = "SYSTEM STOP MERGES default.test;" min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;" with When(f"Insert to MergeTree table {parts_limits} parts"): r = parts_limits sql = stop_merges + min_block + \ "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT %d;" % r clickhouse.clickhouse_query(chi_name, sql, host=selected_svc, ns=kubectl.namespace) # @TODO we need only one query after resolve https://github.com/ClickHouse/ClickHouse/issues/11384 sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace) with And(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 seconds"): time.sleep(prometheus_scrape_interval * 2) sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
def test_distributed_files_to_insert(): delayed_pod, delayed_svc, restarted_pod, restarted_svc = random_pod_choice_for_callbacks() create_distributed_table_on_cluster() # we need 70 delayed files for catch insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 10000' # clickhouse.clickhouse_query( # chi["metadata"]["name"], 'SYSTEM STOP DISTRIBUTED SENDS default.test_distr', # host=delayed_svc, ns=kubectl.namespace # ) files_to_insert_from_metrics = 0 files_to_insert_from_disk = 0 tries = 0 while files_to_insert_from_disk < 50 and tries < 500: kubectl.kubectl( f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1", ok_to_fail=True, ) clickhouse.clickhouse_query(chi["metadata"]["name"], insert_sql, host=delayed_svc, ns=kubectl.namespace) files_to_insert_from_metrics = clickhouse.clickhouse_query( chi["metadata"]["name"], "SELECT value FROM system.metrics WHERE metric='DistributedFilesToInsert'", host=delayed_svc, ns=kubectl.namespace ) files_to_insert_from_metrics = int(files_to_insert_from_metrics) files_to_insert_from_disk = int(kubectl.kubectl( f"exec -n {kubectl.namespace} {delayed_pod} -c clickhouse -- bash -c 'ls -la /var/lib/clickhouse/data/default/test_distr/*/*.bin 2>/dev/null | wc -l'", ok_to_fail=False, )) with When("reboot clickhouse-server pod"): fired = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", True, labels={"hostname": delayed_svc, "chi": chi["metadata"]["name"]}) assert fired, error("can't get ClickHouseDistributedFilesToInsertHigh alert in firing state") # @TODO remove it when https://github.com/ClickHouse/ClickHouse/pull/11220 will merged to docker latest image kubectl.kube_wait_pod_status(restarted_pod, "Running", ns=kubectl.namespace) with Then("check ClickHouseClickHouseDistributedFilesToInsertHigh gone away"): resolved = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", False, labels={"hostname": delayed_svc}) assert resolved, error("can't check ClickHouseDistributedFilesToInsertHigh alert is gone away") drop_distributed_table_on_cluster()
def create_replicated_table_on_cluster(cluster_name='all-replicated'): create_local_sql = 'CREATE TABLE default.test_repl ON CLUSTER \\\"' + cluster_name + '\\\" (event_time DateTime, test UInt64) ENGINE ReplicatedMergeTree(\'/clickhouse/tables/{installation}-{shard}/test_repl\', \'{replica}\') ORDER BY tuple()' clickhouse.clickhouse_query(chi["metadata"]["name"], create_local_sql, timeout=120)
def drop_replicated_table_on_cluster(cluster_name='all-replicated'): drop_repl_sql = 'DROP TABLE default.test_repl ON CLUSTER \\\"' + cluster_name + '\\\"' clickhouse.clickhouse_query(chi["metadata"]["name"], drop_repl_sql, timeout=120)
def create_distributed_table_on_cluster(cluster_name='all-sharded'): create_mergetree_table_on_cluster(cluster_name) create_distr_sql = 'CREATE TABLE default.test_distr ON CLUSTER \\\"' + cluster_name + '\\\" (event_time DateTime, test UInt64) ENGINE Distributed("all-sharded",default, test, test)' clickhouse.clickhouse_query(chi["metadata"]["name"], create_distr_sql, timeout=120)
def drop_distributed_table_on_cluster(cluster_name='all-sharded'): drop_distr_sql = 'DROP TABLE default.test_distr ON CLUSTER \\\"' + cluster_name + '\\\"' clickhouse.clickhouse_query(chi["metadata"]["name"], drop_distr_sql, timeout=120) drop_mergetree_table_on_cluster(cluster_name)
def create_mergetree_table_on_cluster(cluster_name='all-sharded'): create_local_sql = 'CREATE TABLE default.test ON CLUSTER \\\"' + cluster_name + '\\\" (event_time DateTime, test UInt64) ENGINE MergeTree() ORDER BY tuple()' clickhouse.clickhouse_query(chi["metadata"]["name"], create_local_sql, timeout=120)
def test_delayed_and_rejected_insert_and_max_part_count_for_partition_and_low_inserted_rows_per_query(): create_mergetree_table_on_cluster() delayed_pod, delayed_svc, rejected_pod, rejected_svc = random_pod_choice_for_callbacks() prometheus_scrape_interval = 30 # default values in system.merge_tree_settings parts_to_throw_insert = 300 parts_to_delay_insert = 150 chi_name = chi["metadata"]["name"] parts_limits = parts_to_delay_insert selected_svc = delayed_svc def insert_many_parts_to_clickhouse(): stop_merges = "SYSTEM STOP MERGES default.test;" min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;" with When(f"Insert to MergeTree table {parts_limits} parts"): r = parts_limits sql = stop_merges + min_block + \ "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT %d;" % r clickhouse.clickhouse_query(chi_name, sql, host=selected_svc, ns=kubectl.namespace) # @TODO we need only one query after resolve https://github.com/ClickHouse/ClickHouse/issues/11384 sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace) with And(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 seconds"): time.sleep(prometheus_scrape_interval * 2) sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace) insert_many_parts_to_clickhouse() with Then("check ClickHouseDelayedInsertThrottling firing"): fired = wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", True, labels={"hostname": delayed_svc}, time_range="30s", sleep_time=5) assert fired, error("can't get ClickHouseDelayedInsertThrottling alert in firing state") with Then("check ClickHouseMaxPartCountForPartition firing"): fired = wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", True, labels={"hostname": delayed_svc}, time_range="45s", sleep_time=5) assert fired, error("can't get ClickHouseMaxPartCountForPartition alert in firing state") with Then("check ClickHouseLowInsertedRowsPerQuery firing"): fired = wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", True, labels={"hostname": delayed_svc}, time_range="60s", sleep_time=5) assert fired, error("can't get ClickHouseLowInsertedRowsPerQuery alert in firing state") clickhouse.clickhouse_query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace) with Then("check ClickHouseDelayedInsertThrottling gone away"): resolved = wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseDelayedInsertThrottling alert is gone away") with Then("check ClickHouseMaxPartCountForPartition gone away"): resolved = wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseMaxPartCountForPartition alert is gone away") with Then("check ClickHouseLowInsertedRowsPerQuery gone away"): resolved = wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseLowInsertedRowsPerQuery alert is gone away") parts_limits = parts_to_throw_insert selected_svc = rejected_svc insert_many_parts_to_clickhouse() with Then("check ClickHouseRejectedInsert firing"): fired = wait_alert_state("ClickHouseRejectedInsert", "firing", True, labels={"hostname": rejected_svc}, time_range="30s", sleep_time=5) assert fired, error("can't get ClickHouseRejectedInsert alert in firing state") with Then("check ClickHouseRejectedInsert gone away"): resolved = wait_alert_state("ClickHouseRejectedInsert", "firing", False, labels={"hostname": rejected_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseRejectedInsert alert is gone away") clickhouse.clickhouse_query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace) drop_mergetree_table_on_cluster()