def test_replicas_max_abosulute_delay():
stop_replica_pod, stop_replica_svc, insert_pod, insert_svc = random_pod_choice_for_callbacks()
create_replicated_table_on_cluster()
prometheus_scrape_interval = 30
def restart_clickhouse_and_insert_to_replicated_table():
with When(f"stop replica fetches on {stop_replica_svc}"):
sql = "SYSTEM STOP FETCHES default.test_repl"
kubectl.kubectl(
f"exec -n {kubectl.namespace} {stop_replica_pod} -c clickhouse -- clickhouse-client -q \"{sql}\"",
ok_to_fail=True,
)
sql = "INSERT INTO default.test_repl SELECT now(), number FROM numbers(100000)"
kubectl.kubectl(
f"exec -n {kubectl.namespace} {insert_pod} -c clickhouse -- clickhouse-client -q \"{sql}\"",
)
with Then("check ClickHouseReplicasMaxAbsoluteDelay firing"):
fired = wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", True, labels={"hostname": stop_replica_svc},
time_range='60s', sleep_time=prometheus_scrape_interval*2,
callback=restart_clickhouse_and_insert_to_replicated_table)
assert fired, error("can't get ClickHouseReadonlyReplica alert in firing state")
clickhouse.clickhouse_query(
chi["metadata"]["name"],
"SYSTEM START FETCHES; SYSTEM RESTART REPLICAS; SYSTEM SYNC REPLICA default.test_repl", timeout=240
)
with Then("check ClickHouseReplicasMaxAbsoluteDelay gone away"):
resolved = wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", False, labels={"hostname": stop_replica_svc})
assert resolved, error("can't check ClickHouseReplicasMaxAbsoluteDelay alert is gone away")
drop_replicated_table_on_cluster()
def run_queries_with_priority():
sql = ""
for i in range(50):
sql += f"SET priority={i % 20};SELECT uniq(number) FROM numbers(20000000):"
cmd = f"echo \\\"{sql} SELECT 1\\\" | xargs -i'{{}}' --no-run-if-empty -d ':' -P 20 clickhouse-client --time -m -n -q \\\"{{}}\\\""
kubectl.kubectl(f"exec {priority_pod} -- bash -c \"{cmd}\"", timeout=120)
clickhouse.clickhouse_query(
chi["metadata"]["name"],
"SELECT event_time, CurrentMetric_QueryPreempted FROM system.metric_log WHERE CurrentMetric_QueryPreempted > 0",
host=priority_svc,
)
def test_longest_running_query():
long_running_pod, long_running_svc, _, _ = random_pod_choice_for_callbacks()
# 600s trigger + 2*30s - double prometheus scraping interval
clickhouse.clickhouse_query(chi["metadata"]["name"], "SELECT now(),sleepEachRow(1),number FROM system.numbers LIMIT 660",
host=long_running_svc, timeout=670)
with Then("check ClickHouseLongestRunningQuery firing"):
fired = wait_alert_state("ClickHouseLongestRunningQuery", "firing", True, labels={"hostname": long_running_svc},
time_range='30s', sleep_time=5)
assert fired, error("can't get ClickHouseLongestRunningQuery alert in firing state")
with Then("check ClickHouseLongestRunningQuery gone away"):
resolved = wait_alert_state("ClickHouseLongestRunningQuery", "firing", False, labels={"hostname": long_running_svc})
assert resolved, error("can't check ClickHouseLongestRunningQuery alert is gone away")
def reboot_clickhouse_and_distributed_exection():
# we need 70 delayed files for catch
insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 10000'
select_sql = 'SELECT count() FROM default.test_distr'
with Then("reboot clickhouse-server pod"):
kubectl.kubectl(
f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1",
ok_to_fail=True,
)
with And("Insert to distributed table"):
clickhouse.clickhouse_query(chi["metadata"]["name"], insert_sql, host=delayed_pod, ns=kubectl.namespace)
with And("Select from distributed table"):
clickhouse.clickhouse_query_with_error(chi["metadata"]["name"], select_sql, host=delayed_pod,
ns=kubectl.namespace)
def insert_many_parts_to_clickhouse():
stop_merges = "SYSTEM STOP MERGES default.test;"
min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;"
with When(f"Insert to MergeTree table {parts_limits} parts"):
r = parts_limits
sql = stop_merges + min_block + \
"INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT %d;" % r
clickhouse.clickhouse_query(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
# @TODO we need only one query after resolve https://github.com/ClickHouse/ClickHouse/issues/11384
sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;"
clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
with And(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 seconds"):
time.sleep(prometheus_scrape_interval * 2)
sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;"
clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
def test_distributed_files_to_insert():
delayed_pod, delayed_svc, restarted_pod, restarted_svc = random_pod_choice_for_callbacks()
create_distributed_table_on_cluster()
# we need 70 delayed files for catch
insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 10000'
# clickhouse.clickhouse_query(
# chi["metadata"]["name"], 'SYSTEM STOP DISTRIBUTED SENDS default.test_distr',
# host=delayed_svc, ns=kubectl.namespace
# )
files_to_insert_from_metrics = 0
files_to_insert_from_disk = 0
tries = 0
while files_to_insert_from_disk < 50 and tries < 500:
kubectl.kubectl(
f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1",
ok_to_fail=True,
)
clickhouse.clickhouse_query(chi["metadata"]["name"], insert_sql, host=delayed_svc, ns=kubectl.namespace)
files_to_insert_from_metrics = clickhouse.clickhouse_query(
chi["metadata"]["name"], "SELECT value FROM system.metrics WHERE metric='DistributedFilesToInsert'",
host=delayed_svc, ns=kubectl.namespace
)
files_to_insert_from_metrics = int(files_to_insert_from_metrics)
files_to_insert_from_disk = int(kubectl.kubectl(
f"exec -n {kubectl.namespace} {delayed_pod} -c clickhouse -- bash -c 'ls -la /var/lib/clickhouse/data/default/test_distr/*/*.bin 2>/dev/null | wc -l'",
ok_to_fail=False,
))
with When("reboot clickhouse-server pod"):
fired = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", True,
labels={"hostname": delayed_svc, "chi": chi["metadata"]["name"]})
assert fired, error("can't get ClickHouseDistributedFilesToInsertHigh alert in firing state")
# @TODO remove it when https://github.com/ClickHouse/ClickHouse/pull/11220 will merged to docker latest image
kubectl.kube_wait_pod_status(restarted_pod, "Running", ns=kubectl.namespace)
with Then("check ClickHouseClickHouseDistributedFilesToInsertHigh gone away"):
resolved = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", False, labels={"hostname": delayed_svc})
assert resolved, error("can't check ClickHouseDistributedFilesToInsertHigh alert is gone away")
drop_distributed_table_on_cluster()
def create_replicated_table_on_cluster(cluster_name='all-replicated'):
create_local_sql = 'CREATE TABLE default.test_repl ON CLUSTER \\\"' + cluster_name + '\\\" (event_time DateTime, test UInt64) ENGINE ReplicatedMergeTree(\'/clickhouse/tables/{installation}-{shard}/test_repl\', \'{replica}\') ORDER BY tuple()'
clickhouse.clickhouse_query(chi["metadata"]["name"], create_local_sql, timeout=120)
def drop_replicated_table_on_cluster(cluster_name='all-replicated'):
drop_repl_sql = 'DROP TABLE default.test_repl ON CLUSTER \\\"' + cluster_name + '\\\"'
clickhouse.clickhouse_query(chi["metadata"]["name"], drop_repl_sql, timeout=120)
def create_distributed_table_on_cluster(cluster_name='all-sharded'):
create_mergetree_table_on_cluster(cluster_name)
create_distr_sql = 'CREATE TABLE default.test_distr ON CLUSTER \\\"' + cluster_name + '\\\" (event_time DateTime, test UInt64) ENGINE Distributed("all-sharded",default, test, test)'
clickhouse.clickhouse_query(chi["metadata"]["name"], create_distr_sql, timeout=120)
def drop_distributed_table_on_cluster(cluster_name='all-sharded'):
drop_distr_sql = 'DROP TABLE default.test_distr ON CLUSTER \\\"' + cluster_name + '\\\"'
clickhouse.clickhouse_query(chi["metadata"]["name"], drop_distr_sql, timeout=120)
drop_mergetree_table_on_cluster(cluster_name)
def create_mergetree_table_on_cluster(cluster_name='all-sharded'):
create_local_sql = 'CREATE TABLE default.test ON CLUSTER \\\"' + cluster_name + '\\\" (event_time DateTime, test UInt64) ENGINE MergeTree() ORDER BY tuple()'
clickhouse.clickhouse_query(chi["metadata"]["name"], create_local_sql, timeout=120)
def test_delayed_and_rejected_insert_and_max_part_count_for_partition_and_low_inserted_rows_per_query():
create_mergetree_table_on_cluster()
delayed_pod, delayed_svc, rejected_pod, rejected_svc = random_pod_choice_for_callbacks()
prometheus_scrape_interval = 30
# default values in system.merge_tree_settings
parts_to_throw_insert = 300
parts_to_delay_insert = 150
chi_name = chi["metadata"]["name"]
parts_limits = parts_to_delay_insert
selected_svc = delayed_svc
def insert_many_parts_to_clickhouse():
stop_merges = "SYSTEM STOP MERGES default.test;"
min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;"
with When(f"Insert to MergeTree table {parts_limits} parts"):
r = parts_limits
sql = stop_merges + min_block + \
"INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT %d;" % r
clickhouse.clickhouse_query(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
# @TODO we need only one query after resolve https://github.com/ClickHouse/ClickHouse/issues/11384
sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;"
clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
with And(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 seconds"):
time.sleep(prometheus_scrape_interval * 2)
sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;"
clickhouse.clickhouse_query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
insert_many_parts_to_clickhouse()
with Then("check ClickHouseDelayedInsertThrottling firing"):
fired = wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", True, labels={"hostname": delayed_svc}, time_range="30s", sleep_time=5)
assert fired, error("can't get ClickHouseDelayedInsertThrottling alert in firing state")
with Then("check ClickHouseMaxPartCountForPartition firing"):
fired = wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", True, labels={"hostname": delayed_svc}, time_range="45s", sleep_time=5)
assert fired, error("can't get ClickHouseMaxPartCountForPartition alert in firing state")
with Then("check ClickHouseLowInsertedRowsPerQuery firing"):
fired = wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", True, labels={"hostname": delayed_svc}, time_range="60s", sleep_time=5)
assert fired, error("can't get ClickHouseLowInsertedRowsPerQuery alert in firing state")
clickhouse.clickhouse_query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace)
with Then("check ClickHouseDelayedInsertThrottling gone away"):
resolved = wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5)
assert resolved, error("can't check ClickHouseDelayedInsertThrottling alert is gone away")
with Then("check ClickHouseMaxPartCountForPartition gone away"):
resolved = wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5)
assert resolved, error("can't check ClickHouseMaxPartCountForPartition alert is gone away")
with Then("check ClickHouseLowInsertedRowsPerQuery gone away"):
resolved = wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5)
assert resolved, error("can't check ClickHouseLowInsertedRowsPerQuery alert is gone away")
parts_limits = parts_to_throw_insert
selected_svc = rejected_svc
insert_many_parts_to_clickhouse()
with Then("check ClickHouseRejectedInsert firing"):
fired = wait_alert_state("ClickHouseRejectedInsert", "firing", True, labels={"hostname": rejected_svc}, time_range="30s", sleep_time=5)
assert fired, error("can't get ClickHouseRejectedInsert alert in firing state")
with Then("check ClickHouseRejectedInsert gone away"):
resolved = wait_alert_state("ClickHouseRejectedInsert", "firing", False, labels={"hostname": rejected_svc}, sleep_time=5)
assert resolved, error("can't check ClickHouseRejectedInsert alert is gone away")
clickhouse.clickhouse_query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace)
drop_mergetree_table_on_cluster()
