def initialize(chi_file, chi_template_file, chi_name):
prometheus_operator_spec, prometheus_spec, alertmanager_spec = get_prometheus_and_alertmanager_spec(
)
clickhouse_operator_spec, chi = util.install_clickhouse_and_zookeeper(
chi_file, chi_template_file, chi_name)
util.wait_clickhouse_cluster_ready(chi)
return prometheus_operator_spec, prometheus_spec, alertmanager_spec, clickhouse_operator_spec, chi
def test_ch_001(self):
util.require_zookeeper()
quorum_template = "manifests/chit/tpl-clickhouse-21.8.yaml"
chit_data = yaml_manifest.get_manifest_data(
util.get_full_path(quorum_template))
kubectl.launch(f"delete chit {chit_data['metadata']['name']}",
ns=settings.test_namespace,
ok_to_fail=True)
kubectl.create_and_check("manifests/chi/test-ch-001-insert-quorum.yaml", {
"apply_templates": {quorum_template},
"pod_count": 2,
"do_not_delete": 1,
})
chi = yaml_manifest.get_chi_name(
util.get_full_path("manifests/chi/test-ch-001-insert-quorum.yaml"))
chi_data = kubectl.get("chi", ns=settings.test_namespace, name=chi)
util.wait_clickhouse_cluster_ready(chi_data)
host0 = "chi-test-ch-001-insert-quorum-default-0-0"
host1 = "chi-test-ch-001-insert-quorum-default-0-1"
create_table = """
create table t1 on cluster default (a Int8, d Date default today())
Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}')
partition by d order by a
TTL d + interval 5 second
SETTINGS merge_with_ttl_timeout=5""".replace('\r', '').replace('\n', '')
create_mv_table2 = """
create table t2 on cluster default (a Int8)
Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}')
partition by tuple() order by a""".replace('\r', '').replace('\n', '')
create_mv_table3 = """
create table t3 on cluster default (a Int8)
Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}')
partition by tuple() order by a""".replace('\r', '').replace('\n', '')
create_mv2 = "create materialized view t_mv2 on cluster default to t2 as select a from t1"
create_mv3 = "create materialized view t_mv3 on cluster default to t3 as select a from t1"
with Given("Tables t1, t2, t3 and MVs t1->t2, t1-t3 are created"):
clickhouse.query(chi, create_table)
clickhouse.query(chi, create_mv_table2)
clickhouse.query(chi, create_mv_table3)
clickhouse.query(chi, create_mv2)
clickhouse.query(chi, create_mv3)
with When("Add a row to an old partition"):
clickhouse.query(chi,
"insert into t1(a,d) values(6, today()-1)",
host=host0)
with When("Stop fetches for t1 at replica1"):
clickhouse.query(chi, "system stop fetches default.t1", host=host1)
with Then("Wait 10 seconds and the data should be dropped by TTL"):
time.sleep(10)
out = clickhouse.query(chi,
"select count() from t1 where a=6",
host=host0)
assert out == "0", error()
with When("Resume fetches for t1 at replica1"):
clickhouse.query(chi,
"system start fetches default.t1",
host=host1)
time.sleep(5)
with Then("Inserts should resume"):
clickhouse.query(chi,
"insert into t1(a) values(7)",
host=host0)
clickhouse.query(chi, "insert into t1(a) values(1)")
with When("Stop fetches for t2 at replica1"):
clickhouse.query(chi, "system stop fetches default.t2", host=host1)
with Then("Insert should fail since it can not reach the quorum"):
out = clickhouse.query_with_error(
chi, "insert into t1(a) values(2)", host=host0)
assert "Timeout while waiting for quorum" in out, error()
# kubectl(f"exec {host0}-0 -n test -- cp /var/lib//clickhouse/data/default/t2/all_1_1_0/a.mrk2 /var/lib//clickhouse/data/default/t2/all_1_1_0/a.bin")
# with Then("Corrupt data part in t2"):
# kubectl(f"exec {host0}-0 -n test -- sed -i \"s/b/c/\" /var/lib/clickhouse/data/default/t2/all_1_1_0/a.bin")
with When("Resume fetches for t2 at replica1"):
clickhouse.query(chi,
"system start fetches default.t2",
host=host1)
i = 0
while "2" != clickhouse.query(
chi,
"select active_replicas from system.replicas where database='default' and table='t1'",
pod=host0) and i < 10:
with Then("Not ready, wait 5 seconds"):
time.sleep(5)
i += 1
with Then(
"Inserts should fail with an error regarding not satisfied quorum"
):
out = clickhouse.query_with_error(
chi, "insert into t1(a) values(3)", host=host0)
assert "Quorum for previous write has not been satisfied yet" in out, error(
)
with And("Second insert of the same block should pass"):
clickhouse.query(chi,
"insert into t1(a) values(3)",
host=host0)
with And("Insert of the new block should fail"):
out = clickhouse.query_with_error(
chi, "insert into t1(a) values(4)", host=host0)
assert "Quorum for previous write has not been satisfied yet" in out, error(
)
with And(
"Second insert of the same block with 'deduplicate_blocks_in_dependent_materialized_views' setting should fail"
):
out = clickhouse.query_with_error(
chi,
"set deduplicate_blocks_in_dependent_materialized_views=1; insert into t1(a) values(5)",
host=host0)
assert "Quorum for previous write has not been satisfied yet" in out, error(
)
out = clickhouse.query_with_error(
chi,
"select t1.a t1_a, t2.a t2_a from t1 left outer join t2 using (a) order by t1_a settings join_use_nulls=1"
)
note(out)
def test_zookeeper_rescale(self):
"""
test scenario for ZK
CH 1 -> 2 wait complete + ZK 1 -> 3 nowait
CH 2 -> 1 wait complete + ZK 3 -> 1 nowait
CH 1 -> 2 wait complete + ZK 1 -> 3 nowait
"""
def insert_replicated_data(chi, create_tables, insert_tables):
with When(f'create if not exists replicated tables {create_tables}'):
for table in create_tables:
clickhouse.create_table_on_cluster(
chi,
'all-sharded',
f'default.{table}',
f'(id UInt64) ENGINE=ReplicatedMergeTree(\'/clickhouse/tables/default.{table}/{{shard}}\',\'{{replica}}\') ORDER BY (id)',
if_not_exists=True,
)
with When(f'insert tables data {insert_tables}'):
for table in insert_tables:
clickhouse.query(
chi['metadata']['name'],
f'INSERT INTO default.{table} SELECT rand()+number FROM numbers(1000)',
pod="chi-test-cluster-for-zk-default-0-1-0")
def check_zk_root_znode(chi, pod_count, zk_retry=5):
for pod_num in range(pod_count):
out = ""
for i in range(zk_retry):
out = kubectl.launch(
f"exec zookeeper-{pod_num} -- bash -ce './bin/zkCli.sh ls /'",
ns=settings.test_namespace,
ok_to_fail=True)
if "[clickhouse, zookeeper]" in out:
break
else:
with Then(
f"Zookeeper ROOT NODE not ready, wait { (i+1)*3} sec"
):
time.sleep((i + 1) * 3)
assert "[clickhouse, zookeeper]" in out, "Unexpected `zkCli.sh ls /` output"
out = clickhouse.query(
chi["metadata"]["name"],
"SELECT count() FROM system.zookeeper WHERE path='/'")
assert "2" == out.strip(
" \t\r\n"
), f"Unexpected `SELECT count() FROM system.zookeeper WHERE path='/'` output {out}"
def rescale_zk_and_clickhouse(ch_node_count,
zk_node_count,
first_install=False):
zk_manifest = 'zookeeper-1-node-1GB-for-tests-only.yaml' if zk_node_count == 1 else 'zookeeper-3-nodes-1GB-for-tests-only.yaml'
_, chi = util.install_clickhouse_and_zookeeper(
chi_file=
f'manifests/chi/test-cluster-for-zookeeper-{ch_node_count}.yaml',
chi_template_file='manifests/chit/tpl-clickhouse-latest.yaml',
chi_name='test-cluster-for-zk',
zk_manifest=zk_manifest,
clean_ns=first_install,
force_zk_install=True,
zk_install_first=first_install,
make_object_count=False,
)
return chi
with When("Clean exists ClickHouse and Zookeeper"):
kubectl.delete_all_zookeeper(settings.test_namespace)
kubectl.delete_all_chi(settings.test_namespace)
with When("Install CH 1 node ZK 1 node"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
zk_node_count=1,
first_install=True)
util.wait_clickhouse_cluster_ready(chi)
wait_zookeeper_ready(pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(chi,
create_tables=['test_repl1'],
insert_tables=['test_repl1'])
total_iterations = 5
for iteration in range(total_iterations):
with When(f"ITERATION {iteration}"):
with Then("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2,
zk_node_count=3)
wait_zookeeper_ready(pod_count=3)
check_zk_root_znode(chi, pod_count=3)
util.wait_clickhouse_cluster_ready(chi)
insert_replicated_data(
chi,
create_tables=['test_repl2'],
insert_tables=['test_repl1', 'test_repl2'])
with Then("CH 2 -> 1 wait complete + ZK 3 -> 1 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
zk_node_count=1)
wait_zookeeper_ready(pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
insert_replicated_data(
chi,
create_tables=['test_repl3'],
insert_tables=['test_repl1', 'test_repl2', 'test_repl3'])
with When("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2, zk_node_count=3)
check_zk_root_znode(chi, pod_count=3)
with Then('check data in tables'):
for table, exptected_rows in {
"test_repl1": str(1000 + 2000 * total_iterations),
"test_repl2": str(2000 * total_iterations),
"test_repl3": str(1000 * total_iterations)
}.items():
actual_rows = clickhouse.query(
chi['metadata']['name'],
f'SELECT count() FROM default.{table}',
pod="chi-test-cluster-for-zk-default-0-1-0")
assert actual_rows == exptected_rows, f"Invalid rows counter after inserts {table} expected={exptected_rows} actual={actual_rows}"
with Then('drop all created tables'):
for i in range(3):
clickhouse.drop_table_on_cluster(chi, 'all-sharded',
f'default.test_repl{i+1}')
def test_keeper_outline(
self,
keeper_type="zookeeper",
pod_for_insert_data="chi-test-cluster-for-zk-default-0-1-0",
keeper_manifest_1_node='zookeeper-1-node-1GB-for-tests-only.yaml',
keeper_manifest_3_node='zookeeper-3-nodes-1GB-for-tests-only.yaml',
):
"""
test scenario for Zoo/Clickhouse Keeper
CH 1 -> 2 wait complete + Keeper 1 -> 3 nowait
CH 2 -> 1 wait complete + Keeper 3 -> 1 nowait
CH 1 -> 2 wait complete + Keeper 1 -> 3 nowait
"""
def insert_replicated_data(chi, create_tables, insert_tables):
with When(f'create if not exists replicated tables {create_tables}'):
for table in create_tables:
clickhouse.create_table_on_cluster(
chi,
'all-sharded',
f'default.{table}',
f'(id UInt64) ENGINE=ReplicatedMergeTree(\'/clickhouse/tables/default.{table}/{{shard}}\',\'{{replica}}\') ORDER BY (id)',
if_not_exists=True,
)
with When(f'insert tables data {insert_tables}'):
for table in insert_tables:
clickhouse.query(
chi['metadata']['name'],
f'INSERT INTO default.{table} SELECT rand()+number FROM numbers(1000)',
pod=pod_for_insert_data)
def check_zk_root_znode(chi, pod_count, retry_count=5):
for pod_num in range(pod_count):
out = ""
expected_out = ""
for i in range(retry_count):
if keeper_type == "zookeeper-operator":
expected_out = "[clickhouse, zookeeper, zookeeper-operator]"
keeper_cmd = './bin/zkCli.sh ls /'
pod_prefix = "zookeeper"
elif keeper_type == "zookeeper":
expected_out = "[clickhouse, zookeeper]"
keeper_cmd = './bin/zkCli.sh ls /'
pod_prefix = "zookeeper"
else:
expected_out = "clickhouse"
keeper_cmd = "if [[ ! $(command -v zookeepercli) ]]; then "
keeper_cmd += "wget -q -O /tmp/zookeepercli.deb https://github.com/outbrain/zookeepercli/releases/download/v1.0.12/zookeepercli_1.0.12_amd64.deb; "
keeper_cmd += "dpkg -i /tmp/zookeepercli.deb; "
keeper_cmd += "fi; "
keeper_cmd += "zookeepercli -servers 127.0.0.1:2181 -c ls /"
pod_prefix = "clickhouse-keeper"
out = kubectl.launch(
f"exec {pod_prefix}-{pod_num} -- bash -ce '{keeper_cmd}'",
ns=settings.test_namespace,
ok_to_fail=True)
if expected_out in out:
break
else:
with Then(
f"{keeper_type} ROOT NODE not ready, wait {(i + 1) * 3} sec"
):
time.sleep((i + 1) * 3)
assert expected_out in out, f"Unexpected {keeper_type} `ls /` output"
out = clickhouse.query(
chi["metadata"]["name"],
"SELECT count() FROM system.zookeeper WHERE path='/'")
expected_out = {
"zookeeper": "2",
"zookeeper-operator": "3",
"clickhouse-keeper": "1",
}
assert expected_out[keeper_type] == out.strip(
" \t\r\n"
), f"Unexpected `SELECT count() FROM system.zookeeper WHERE path='/'` output {out}"
def rescale_zk_and_clickhouse(ch_node_count,
keeper_node_count,
first_install=False):
keeper_manifest = keeper_manifest_1_node if keeper_node_count == 1 else keeper_manifest_3_node
_, chi = util.install_clickhouse_and_keeper(
chi_file=
f'manifests/chi/test-cluster-for-{keeper_type}-{ch_node_count}.yaml',
chi_template_file='manifests/chit/tpl-clickhouse-latest.yaml',
chi_name='test-cluster-for-zk',
keeper_manifest=keeper_manifest,
keeper_type=keeper_type,
clean_ns=first_install,
force_keeper_install=True,
keeper_install_first=first_install,
make_object_count=False,
)
return chi
with When("Clean exists ClickHouse Keeper and ZooKeeper"):
kubectl.delete_all_chi(settings.test_namespace)
kubectl.delete_all_keeper(settings.test_namespace)
with When("Install CH 1 node ZK 1 node"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
keeper_node_count=1,
first_install=True)
util.wait_clickhouse_cluster_ready(chi)
wait_keeper_ready(keeper_type=keeper_type, pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(chi,
create_tables=['test_repl1'],
insert_tables=['test_repl1'])
total_iterations = 3
for iteration in range(total_iterations):
with When(f"ITERATION {iteration}"):
with Then("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2,
keeper_node_count=3)
wait_keeper_ready(keeper_type=keeper_type, pod_count=3)
check_zk_root_znode(chi, pod_count=3)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(
chi,
create_tables=['test_repl2'],
insert_tables=['test_repl1', 'test_repl2'])
with Then("CH 2 -> 1 wait complete + ZK 3 -> 1 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=1,
keeper_node_count=1)
wait_keeper_ready(keeper_type=keeper_type, pod_count=1)
check_zk_root_znode(chi, pod_count=1)
util.wait_clickhouse_cluster_ready(chi)
wait_clickhouse_no_readonly_replicas(chi)
insert_replicated_data(
chi,
create_tables=['test_repl3'],
insert_tables=['test_repl1', 'test_repl2', 'test_repl3'])
with When("CH 1 -> 2 wait complete + ZK 1 -> 3 nowait"):
chi = rescale_zk_and_clickhouse(ch_node_count=2, keeper_node_count=3)
check_zk_root_znode(chi, pod_count=3)
with Then('check data in tables'):
for table_name, exptected_rows in {
"test_repl1": str(1000 + 2000 * total_iterations),
"test_repl2": str(2000 * total_iterations),
"test_repl3": str(1000 * total_iterations)
}.items():
actual_rows = clickhouse.query(
chi['metadata']['name'],
f'SELECT count() FROM default.{table_name}',
pod="chi-test-cluster-for-zk-default-0-1-0")
assert actual_rows == exptected_rows, f"Invalid rows counter after inserts {table_name} expected={exptected_rows} actual={actual_rows}"
with Then('drop all created tables'):
for i in range(3):
clickhouse.drop_table_on_cluster(chi, 'all-sharded',
f'default.test_repl{i + 1}')