def test_detached_parts(self, prometheus_operator_spec, clickhouse_operator_spec, chi):
clickhouse.create_table_on_cluster(chi)
detached_pod, detached_svc, _, _ = alerts.random_pod_choice_for_callbacks(chi)
def create_part_and_detach():
clickhouse.query(chi["metadata"]["name"], "INSERT INTO default.test SELECT now(), number FROM numbers(100)", pod=detached_pod)
part_name = clickhouse.query(
chi["metadata"]["name"],
sql="SELECT name FROM system.parts WHERE database='default' AND table='test' ORDER BY modification_time DESC LIMIT 1",
pod=detached_pod
)
clickhouse.query(chi["metadata"]["name"], f"ALTER TABLE default.test DETACH PART '{part_name}'", pod=detached_pod)
def attach_all_parts():
detached_parts = clickhouse.query(chi["metadata"]["name"], "SELECT name FROM system.detached_parts WHERE database='default' AND table='test' AND reason=''", pod=detached_pod)
all_parts = ""
for part in detached_parts.splitlines():
all_parts += f"ALTER TABLE default.test ATTACH PART '{part}';"
if all_parts.strip() != "":
clickhouse.query(chi["metadata"]["name"], all_parts, pod=detached_pod)
with When("check ClickHouseDetachedParts firing"):
fired = alerts.wait_alert_state("ClickHouseDetachedParts", "firing", True,
labels={"hostname": detached_svc, "chi": chi["metadata"]["name"]}, time_range='30s',
callback=create_part_and_detach)
assert fired, error("can't get ClickHouseDetachedParts alert in firing state")
with Then("check ClickHouseDetachedParts gone away"):
resolved = alerts.wait_alert_state("ClickHouseDetachedParts", "firing", False,
labels={"hostname": detached_svc}, callback=attach_all_parts)
assert resolved, error("can't check ClickHouseDetachedParts alert is gone away")
clickhouse.drop_table_on_cluster(chi)
def test_read_only_replica(self, prometheus_operator_spec, clickhouse_operator_spec, chi):
read_only_pod, read_only_svc, other_pod, other_svc = alerts.random_pod_choice_for_callbacks(chi)
chi_name = chi["metadata"]["name"]
clickhouse.create_table_on_cluster(
chi,
'all-replicated', 'default.test_repl',
'(event_time DateTime, test UInt64) ' +
'ENGINE ReplicatedMergeTree(\'/clickhouse/tables/{installation}-{shard}/test_repl\', \'{replica}\') ORDER BY tuple()'
)
def restart_zookeeper():
kubectl.launch(
f"exec -n {kubectl.namespace} zookeeper-0 -- sh -c \"kill 1\"",
ok_to_fail=True,
)
clickhouse.query_with_error(chi_name, "INSERT INTO default.test_repl VALUES(now(),rand())", host=read_only_svc)
with Then("check ClickHouseReadonlyReplica firing"):
fired = alerts.wait_alert_state("ClickHouseReadonlyReplica", "firing", True, labels={"hostname": read_only_svc},
time_range='30s', sleep_time=settings.prometheus_scrape_interval, callback=restart_zookeeper)
assert fired, error("can't get ClickHouseReadonlyReplica alert in firing state")
with Then("check ClickHouseReadonlyReplica gone away"):
resolved = alerts.wait_alert_state("ClickHouseReadonlyReplica", "firing", False, labels={"hostname": read_only_svc})
assert resolved, error("can't check ClickHouseReadonlyReplica alert is gone away")
kubectl.wait_pod_status("zookeeper-0", "Running", ns=kubectl.namespace)
kubectl.wait_jsonpath("pod", "zookeeper-0", "{.status.containerStatuses[0].ready}", "true",
ns=kubectl.namespace)
for i in range(11):
zookeeper_status = kubectl.launch(
f"exec -n {kubectl.namespace} zookeeper-0 -- sh -c \"echo ruok | nc 127.0.0.1 2181\"", ok_to_fail=True
)
if "imok" in zookeeper_status:
break
elif i == 10:
fail(f"invalid zookeeper status after {i} retries")
with Then("zookeper is not ready, wait 2 seconds"):
time.sleep(2)
clickhouse.query_with_error(
chi_name, "SYSTEM RESTART REPLICAS; SYSTEM SYNC REPLICA default.test_repl",
host=read_only_svc, timeout=240
)
clickhouse.query_with_error(
chi_name, "SYSTEM RESTART REPLICAS; SYSTEM SYNC REPLICA default.test_repl",
host=other_svc, timeout=240
)
clickhouse.drop_table_on_cluster(chi, 'all-replicated', 'default.test_repl')
def test_replicas_max_absolute_delay(self, prometheus_operator_spec, clickhouse_operator_spec, chi):
stop_replica_pod, stop_replica_svc, insert_pod, insert_svc = alerts.random_pod_choice_for_callbacks(chi)
clickhouse.create_table_on_cluster(
chi,
'all-replicated', 'default.test_repl',
'(event_time DateTime, test UInt64) ' +
'ENGINE ReplicatedMergeTree(\'/clickhouse/tables/{installation}-{shard}/test_repl\', \'{replica}\') ORDER BY tuple()'
)
prometheus_scrape_interval = 15
def restart_clickhouse_and_insert_to_replicated_table():
with When(f"stop replica fetches on {stop_replica_svc}"):
sql = "SYSTEM STOP FETCHES default.test_repl"
kubectl.launch(
f"exec -n {kubectl.namespace} {stop_replica_pod} -c clickhouse-pod -- clickhouse-client -q \"{sql}\"",
ok_to_fail=True, timeout=600,
)
sql = "INSERT INTO default.test_repl SELECT now(), number FROM numbers(100000)"
kubectl.launch(
f"exec -n {kubectl.namespace} {insert_pod} -c clickhouse-pod -- clickhouse-client -q \"{sql}\"",
)
with Then("check ClickHouseReplicasMaxAbsoluteDelay firing"):
fired = alerts.wait_alert_state(
"ClickHouseReplicasMaxAbsoluteDelay", "firing", True, labels={"hostname": stop_replica_svc},
time_range='60s', sleep_time=prometheus_scrape_interval * 2,
callback=restart_clickhouse_and_insert_to_replicated_table
)
assert fired, error("can't get ClickHouseReadonlyReplica alert in firing state")
clickhouse.query(
chi["metadata"]["name"],
"SYSTEM START FETCHES; SYSTEM RESTART REPLICAS; SYSTEM SYNC REPLICA default.test_repl",
host=stop_replica_svc, timeout=240
)
with Then("check ClickHouseReplicasMaxAbsoluteDelay gone away"):
resolved = alerts.wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", False, labels={"hostname": stop_replica_svc})
assert resolved, error("can't check ClickHouseReplicasMaxAbsoluteDelay alert is gone away")
clickhouse.drop_table_on_cluster(chi, 'all-replicated', 'default.test_repl')
def test_insert_related_alerts(self, prometheus_operator_spec, clickhouse_operator_spec, chi):
clickhouse.create_table_on_cluster(chi)
delayed_pod, delayed_svc, rejected_pod, rejected_svc = alerts.random_pod_choice_for_callbacks(chi)
prometheus_scrape_interval = settings.prometheus_scrape_interval
# default values in system.merge_tree_settings
parts_to_throw_insert = 300
parts_to_delay_insert = 150
chi_name = chi["metadata"]["name"]
parts_limits = parts_to_delay_insert
selected_svc = delayed_svc
def insert_many_parts_to_clickhouse():
stop_merges = "SYSTEM STOP MERGES default.test;"
min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;"
with When(f"Insert to MergeTree table {parts_limits} parts"):
r = parts_limits
sql = stop_merges + min_block + f"INSERT INTO default.test(event_time, test) SELECT now(),number FROM system.numbers LIMIT {r};"
clickhouse.query(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;"
clickhouse.query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
with Then(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 sec"):
time.sleep(prometheus_scrape_interval * 2)
with Then("after 21.8 InsertedRows include system.* rows"):
for i in range(35):
sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;"
clickhouse.query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
insert_many_parts_to_clickhouse()
with Then("check ClickHouseDelayedInsertThrottling firing"):
fired = alerts.wait_alert_state(
"ClickHouseDelayedInsertThrottling", "firing", True, labels={"hostname": delayed_svc}, time_range="60s"
)
assert fired, error("can't get ClickHouseDelayedInsertThrottling alert in firing state")
with Then("check ClickHouseMaxPartCountForPartition firing"):
fired = alerts.wait_alert_state(
"ClickHouseMaxPartCountForPartition", "firing", True, labels={"hostname": delayed_svc}, time_range="90s"
)
assert fired, error("can't get ClickHouseMaxPartCountForPartition alert in firing state")
with Then("check ClickHouseLowInsertedRowsPerQuery firing"):
fired = alerts.wait_alert_state(
"ClickHouseLowInsertedRowsPerQuery", "firing", True, labels={"hostname": delayed_svc}, time_range="120s",
)
assert fired, error("can't get ClickHouseLowInsertedRowsPerQuery alert in firing state")
clickhouse.query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace)
with Then("check ClickHouseDelayedInsertThrottling gone away"):
resolved = alerts.wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", False, labels={"hostname": delayed_svc})
assert resolved, error("can't check ClickHouseDelayedInsertThrottling alert is gone away")
with Then("check ClickHouseMaxPartCountForPartition gone away"):
resolved = alerts.wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", False, labels={"hostname": delayed_svc})
assert resolved, error("can't check ClickHouseMaxPartCountForPartition alert is gone away")
with Then("check ClickHouseLowInsertedRowsPerQuery gone away"):
resolved = alerts.wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", False, labels={"hostname": delayed_svc})
assert resolved, error("can't check ClickHouseLowInsertedRowsPerQuery alert is gone away")
parts_limits = parts_to_throw_insert
selected_svc = rejected_svc
insert_many_parts_to_clickhouse()
with Then("check ClickHouseRejectedInsert firing"):
fired = alerts.wait_alert_state("ClickHouseRejectedInsert", "firing", True, labels={"hostname": rejected_svc}, time_range="30s",
sleep_time=settings.prometheus_scrape_interval)
assert fired, error("can't get ClickHouseRejectedInsert alert in firing state")
with Then("check ClickHouseRejectedInsert gone away"):
resolved = alerts.wait_alert_state("ClickHouseRejectedInsert", "firing", False, labels={"hostname": rejected_svc})
assert resolved, error("can't check ClickHouseRejectedInsert alert is gone away")
clickhouse.query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace)
clickhouse.drop_table_on_cluster(chi)
def test_zookeeper_rescale(self):
"""
test scenario for ZK
CH 1 -> 2 wait complete + ZK 1 -> 3 nowait
CH 2 -> 1 wait complete + ZK 3 -> 1 nowait
CH 1 -> 2 wait complete + ZK 1 -> 3 nowait
"""
def insert_replicated_data(chi, create_tables, insert_tables):
with When(f'create if not exists replicated tables {create_tables}'):
for table in create_tables:
clickhouse.create_table_on_cluster(
chi,
'all-sharded',
f'default.{table}',
f'(id UInt64) ENGINE=ReplicatedMergeTree(\'/clickhouse/tables/default.{table}/{{shard}}\',\'{{replica}}\') ORDER BY (id)',
if_not_exists=True,
)
with When(f'insert tables data {insert_tables}'):
for table in insert_tables:
clickhouse.query(
chi['metadata']['name'],
f'INSERT INTO default.{table} SELECT rand()+number FROM numbers(1000)',
pod="chi-test-cluster-for-zk-default-0-1-0")
def check_zk_root_znode(chi, pod_count, zk_retry=5):
for pod_num in range(pod_count):
out = ""
for i in range(zk_retry):
out = kubectl.launch(
f"exec zookeeper-{pod_num} -- bash -ce './bin/zkCli.sh ls /'",
ns=settings.test_namespace,
ok_to_fail=True)
if "[clickhouse, zookeeper]" in out:
break
else:
with Then(
f"Zookeeper ROOT NODE not ready, wait { (i+1)*3} sec"
):
time.sleep((i + 1) * 3)
assert "[clickhouse, zookeeper]" in out, "Unexpected `zkCli.sh ls /` output"
out = clickhouse.query(
chi["metadata"]["name"],
"SELECT count() FROM system.zookeeper WHERE path='/'")
assert "2" == out.strip(
" \t\r\n"
), f"Unexpected `SELECT count() FROM system.zookeeper WHERE path='/'` output {out}"
def rescale_zk_and_clickhouse(ch_node_count,
zk_node_count,
first_install=False):
zk_manifest = 'zookeeper-1-node-1GB-for-tests-only.yaml' if zk_node_count == 1 else 'zookeeper-3-nodes-1GB-for-tests-only.yaml'
_, chi = util.install_clickhouse_and_zookeeper(
chi_file=
f'manifests/chi/test-cluster-for-zookeeper-{ch_node_count}.yaml',
chi_template_file='manifests/chit/tpl-clickhouse-latest.yaml',
chi_name='test-cluster-for-zk',
zk_manifest=zk_manifest,
clean_ns=first_install,
force_zk_install=True,
zk_install_first=first_install,
make_object_count=False,
)
return chi
with When("Clean exists ClickHouse and Zookeeper"):
kubectl.delete_all_zookeeper(settings.test_namespace)
kubectl.delete_all_chi(settings.test_namespace)
with When("Install CH 1 node ZK 1 node"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
zk_node_count=1,
first_install=True)
util.wait_clickhouse_cluster_ready(chi)
wait_zookeeper_ready(pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(chi,
create_tables=['test_repl1'],
insert_tables=['test_repl1'])
total_iterations = 5
for iteration in range(total_iterations):
with When(f"ITERATION {iteration}"):
with Then("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2,
zk_node_count=3)
wait_zookeeper_ready(pod_count=3)
check_zk_root_znode(chi, pod_count=3)
util.wait_clickhouse_cluster_ready(chi)
insert_replicated_data(
chi,
create_tables=['test_repl2'],
insert_tables=['test_repl1', 'test_repl2'])
with Then("CH 2 -> 1 wait complete + ZK 3 -> 1 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
zk_node_count=1)
wait_zookeeper_ready(pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
insert_replicated_data(
chi,
create_tables=['test_repl3'],
insert_tables=['test_repl1', 'test_repl2', 'test_repl3'])
with When("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2, zk_node_count=3)
check_zk_root_znode(chi, pod_count=3)
with Then('check data in tables'):
for table, exptected_rows in {
"test_repl1": str(1000 + 2000 * total_iterations),
"test_repl2": str(2000 * total_iterations),
"test_repl3": str(1000 * total_iterations)
}.items():
actual_rows = clickhouse.query(
chi['metadata']['name'],
f'SELECT count() FROM default.{table}',
pod="chi-test-cluster-for-zk-default-0-1-0")
assert actual_rows == exptected_rows, f"Invalid rows counter after inserts {table} expected={exptected_rows} actual={actual_rows}"
with Then('drop all created tables'):
for i in range(3):
clickhouse.drop_table_on_cluster(chi, 'all-sharded',
f'default.test_repl{i+1}')
def test_keeper_outline(
self,
keeper_type="zookeeper",
pod_for_insert_data="chi-test-cluster-for-zk-default-0-1-0",
keeper_manifest_1_node='zookeeper-1-node-1GB-for-tests-only.yaml',
keeper_manifest_3_node='zookeeper-3-nodes-1GB-for-tests-only.yaml',
):
"""
test scenario for Zoo/Clickhouse Keeper
CH 1 -> 2 wait complete + Keeper 1 -> 3 nowait
CH 2 -> 1 wait complete + Keeper 3 -> 1 nowait
CH 1 -> 2 wait complete + Keeper 1 -> 3 nowait
"""
def insert_replicated_data(chi, create_tables, insert_tables):
with When(f'create if not exists replicated tables {create_tables}'):
for table in create_tables:
clickhouse.create_table_on_cluster(
chi,
'all-sharded',
f'default.{table}',
f'(id UInt64) ENGINE=ReplicatedMergeTree(\'/clickhouse/tables/default.{table}/{{shard}}\',\'{{replica}}\') ORDER BY (id)',
if_not_exists=True,
)
with When(f'insert tables data {insert_tables}'):
for table in insert_tables:
clickhouse.query(
chi['metadata']['name'],
f'INSERT INTO default.{table} SELECT rand()+number FROM numbers(1000)',
pod=pod_for_insert_data)
def check_zk_root_znode(chi, pod_count, retry_count=5):
for pod_num in range(pod_count):
out = ""
expected_out = ""
for i in range(retry_count):
if keeper_type == "zookeeper-operator":
expected_out = "[clickhouse, zookeeper, zookeeper-operator]"
keeper_cmd = './bin/zkCli.sh ls /'
pod_prefix = "zookeeper"
elif keeper_type == "zookeeper":
expected_out = "[clickhouse, zookeeper]"
keeper_cmd = './bin/zkCli.sh ls /'
pod_prefix = "zookeeper"
else:
expected_out = "clickhouse"
keeper_cmd = "if [[ ! $(command -v zookeepercli) ]]; then "
keeper_cmd += "wget -q -O /tmp/zookeepercli.deb https://github.com/outbrain/zookeepercli/releases/download/v1.0.12/zookeepercli_1.0.12_amd64.deb; "
keeper_cmd += "dpkg -i /tmp/zookeepercli.deb; "
keeper_cmd += "fi; "
keeper_cmd += "zookeepercli -servers 127.0.0.1:2181 -c ls /"
pod_prefix = "clickhouse-keeper"
out = kubectl.launch(
f"exec {pod_prefix}-{pod_num} -- bash -ce '{keeper_cmd}'",
ns=settings.test_namespace,
ok_to_fail=True)
if expected_out in out:
break
else:
with Then(
f"{keeper_type} ROOT NODE not ready, wait {(i + 1) * 3} sec"
):
time.sleep((i + 1) * 3)
assert expected_out in out, f"Unexpected {keeper_type} `ls /` output"
out = clickhouse.query(
chi["metadata"]["name"],
"SELECT count() FROM system.zookeeper WHERE path='/'")
expected_out = {
"zookeeper": "2",
"zookeeper-operator": "3",
"clickhouse-keeper": "1",
}
assert expected_out[keeper_type] == out.strip(
" \t\r\n"
), f"Unexpected `SELECT count() FROM system.zookeeper WHERE path='/'` output {out}"
def rescale_zk_and_clickhouse(ch_node_count,
keeper_node_count,
first_install=False):
keeper_manifest = keeper_manifest_1_node if keeper_node_count == 1 else keeper_manifest_3_node
_, chi = util.install_clickhouse_and_keeper(
chi_file=
f'manifests/chi/test-cluster-for-{keeper_type}-{ch_node_count}.yaml',
chi_template_file='manifests/chit/tpl-clickhouse-latest.yaml',
chi_name='test-cluster-for-zk',
keeper_manifest=keeper_manifest,
keeper_type=keeper_type,
clean_ns=first_install,
force_keeper_install=True,
keeper_install_first=first_install,
make_object_count=False,
)
return chi
with When("Clean exists ClickHouse Keeper and ZooKeeper"):
kubectl.delete_all_chi(settings.test_namespace)
kubectl.delete_all_keeper(settings.test_namespace)
with When("Install CH 1 node ZK 1 node"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
keeper_node_count=1,
first_install=True)
util.wait_clickhouse_cluster_ready(chi)
wait_keeper_ready(keeper_type=keeper_type, pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(chi,
create_tables=['test_repl1'],
insert_tables=['test_repl1'])
total_iterations = 3
for iteration in range(total_iterations):
with When(f"ITERATION {iteration}"):
with Then("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2,
keeper_node_count=3)
wait_keeper_ready(keeper_type=keeper_type, pod_count=3)
check_zk_root_znode(chi, pod_count=3)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(
chi,
create_tables=['test_repl2'],
insert_tables=['test_repl1', 'test_repl2'])
with Then("CH 2 -> 1 wait complete + ZK 3 -> 1 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
keeper_node_count=1)
wait_keeper_ready(keeper_type=keeper_type, pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(
chi,
create_tables=['test_repl3'],
insert_tables=['test_repl1', 'test_repl2', 'test_repl3'])
with When("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2, keeper_node_count=3)
check_zk_root_znode(chi, pod_count=3)
with Then('check data in tables'):
for table_name, exptected_rows in {
"test_repl1": str(1000 + 2000 * total_iterations),
"test_repl2": str(2000 * total_iterations),
"test_repl3": str(1000 * total_iterations)
}.items():
actual_rows = clickhouse.query(
chi['metadata']['name'],
f'SELECT count() FROM default.{table_name}',
pod="chi-test-cluster-for-zk-default-0-1-0")
assert actual_rows == exptected_rows, f"Invalid rows counter after inserts {table_name} expected={exptected_rows} actual={actual_rows}"
with Then('drop all created tables'):
for i in range(3):
clickhouse.drop_table_on_cluster(chi, 'all-sharded',
f'default.test_repl{i + 1}')