def test_examples01_1(self): kubectl.create_and_check( manifest= "../../docs/chi-examples/01-simple-layout-01-1shard-1repl.yaml", check={"object_counts": { "statefulset": 1, "pod": 1, "service": 2, }})
def test_examples02_1(self): kubectl.create_and_check( manifest= "../../docs/chi-examples/03-persistent-volume-01-default-volume.yaml", check={ "pod_count": 1, "pod_volumes": { "/var/lib/clickhouse", "/var/log/clickhouse-server", }, })
def test_examples02_2(self): kubectl.create_and_check( manifest= "../../docs/chi-examples/03-persistent-volume-02-pod-template.yaml", check={ "pod_count": 1, "pod_image": "clickhouse/clickhouse-server:22.3", "pod_volumes": { "/var/lib/clickhouse", "/var/log/clickhouse-server", }, })
def test_version_changed(self, prometheus_operator_spec, clickhouse_operator_spec, chi): changed_pod, changed_svc, _, _ = alerts.random_pod_choice_for_callbacks(chi) with When("apply changed settings"): kubectl.create_and_check( manifest="manifests/chi/test-cluster-for-alerts-changed-settings.yaml", check={ "apply_templates": [ "manifests/chit/tpl-clickhouse-stable.yaml", "manifests/chit/tpl-persistent-volume-100Mi.yaml" ], "object_counts": { "statefulset": 2, "pod": 2, "service": 3, }, "do_not_delete": 1 } ) prometheus_scrape_interval = 15 with Then(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 sec"): time.sleep(prometheus_scrape_interval * 2) with Then("check ClickHouseVersionChanged firing"): fired = alerts.wait_alert_state( "ClickHouseVersionChanged", "firing", True, labels={"hostname": changed_svc}, time_range="30s", sleep_time=settings.prometheus_scrape_interval ) assert fired, error("can't get ClickHouseVersionChanged alert in firing state") with When("rollback changed settings"): kubectl.create_and_check( manifest="manifests/chi/test-cluster-for-alerts.yaml", check={ "apply_templates": [ "manifests/chit/tpl-clickhouse-latest.yaml", "manifests/chit/tpl-clickhouse-alerts.yaml", "manifests/chit/tpl-persistent-volume-100Mi.yaml" ], "object_counts": { "statefulset": 2, "pod": 2, "service": 3, }, "do_not_delete": 1 } ) with Then("check ClickHouseVersionChanged gone away"): resolved = alerts.wait_alert_state("ClickHouseVersionChanged", "firing", False, labels={"hostname": changed_svc}, sleep_time=30) assert resolved, error("can't check ClickHouseVersionChanged alert is gone away")
def install_clickhouse_and_keeper(chi_file, chi_template_file, chi_name, keeper_type='zookeeper', keeper_manifest='', force_keeper_install=False, clean_ns=True, keeper_install_first=True, make_object_count=True): if keeper_manifest == '': if keeper_type == 'zookeeper': keeper_manifest = 'zookeeper-1-node-1GB-for-tests-only.yaml' if keeper_type == 'clickhouse-keeper': keeper_manifest = 'clickhouse-keeper-1-node-256M-for-test-only.yaml' if keeper_type == 'zookeeper-operator': keeper_manifest = 'zookeeper-operator-1-node.yaml' with Given("install zookeeper/clickhouse-keeper + clickhouse"): if clean_ns: kubectl.delete_ns(settings.test_namespace, ok_to_fail=True, timeout=600) kubectl.create_ns(settings.test_namespace) # when create clickhouse, need install ZK before CH if keeper_install_first: require_keeper(keeper_type=keeper_type, keeper_manifest=keeper_manifest, force_install=force_keeper_install) chi_manifest_data = yaml_manifest.get_manifest_data(get_full_path(chi_file)) layout = chi_manifest_data["spec"]["configuration"]["clusters"][0]["layout"] expected_nodes = 1 * layout["shardsCount"] * layout["replicasCount"] check = { "apply_templates": [ chi_template_file, "manifests/chit/tpl-persistent-volume-100Mi.yaml" ], "do_not_delete": 1 } if make_object_count: check["object_counts"] = { "statefulset": expected_nodes, "pod": expected_nodes, "service": expected_nodes + 1, } kubectl.create_and_check( manifest=chi_file, check=check, ) clickhouse_operator_spec = kubectl.get( "pod", name="", ns=settings.operator_namespace, label="-l app=clickhouse-operator" ) chi = kubectl.get("chi", ns=settings.test_namespace, name=chi_name) # when re-scale clickhouse, need install Keeper after CH to follow ACM logic if not keeper_install_first: require_keeper(keeper_type=keeper_type, keeper_manifest=keeper_manifest, force_install=force_keeper_install) return clickhouse_operator_spec, chi
def apply_fake_backup(message): with Given(message): kubectl.create_and_check( manifest="manifests/chi/test-cluster-for-backups-fake.yaml", check={ "apply_templates": [ "manifests/chit/tpl-clickhouse-backups-fake.yaml", "manifests/chit/tpl-persistent-volume-100Mi.yaml" ], "object_counts": { "statefulset": 2, "pod": 2, "service": 3, }, "do_not_delete": 1 })
def test_ch_002(self): kubectl.create_and_check( "manifests/chi/test-ch-002-row-level.yaml", { "apply_templates": {"manifests/chit/tpl-clickhouse-21.8.yaml"}, "do_not_delete": 1, }) chi = "test-ch-002-row-level" create_table = """create table test (d Date default today(), team LowCardinality(String), user String) Engine = MergeTree() PARTITION BY d ORDER BY d;""" with When("Create test table"): clickhouse.query(chi, create_table) with And("Insert some data"): clickhouse.query( chi, "INSERT INTO test(team, user) values('team1', 'user1'),('team2', 'user2'),('team3', 'user3'),('team4', 'user4')" ) with Then( "Make another query for different users. It should be restricted to corresponding team by row-level security" ): for user in ['user1', 'user2', 'user3', 'user4']: out = clickhouse.query(chi, "select user from test", user=user, pwd=user) assert out == user, error() with Then( "Make a count() query for different users. It should be restricted to corresponding team by row-level security" ): for user in ['user1', 'user2', 'user3', 'user4']: out = clickhouse.query(chi, "select count() from test", user=user, pwd=user) assert out == "1", error() kubectl.delete_chi(chi)
def test_ch_001(self): util.require_zookeeper() quorum_template = "manifests/chit/tpl-clickhouse-21.8.yaml" chit_data = yaml_manifest.get_manifest_data( util.get_full_path(quorum_template)) kubectl.launch(f"delete chit {chit_data['metadata']['name']}", ns=settings.test_namespace, ok_to_fail=True) kubectl.create_and_check("manifests/chi/test-ch-001-insert-quorum.yaml", { "apply_templates": {quorum_template}, "pod_count": 2, "do_not_delete": 1, }) chi = yaml_manifest.get_chi_name( util.get_full_path("manifests/chi/test-ch-001-insert-quorum.yaml")) chi_data = kubectl.get("chi", ns=settings.test_namespace, name=chi) util.wait_clickhouse_cluster_ready(chi_data) host0 = "chi-test-ch-001-insert-quorum-default-0-0" host1 = "chi-test-ch-001-insert-quorum-default-0-1" create_table = """ create table t1 on cluster default (a Int8, d Date default today()) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by d order by a TTL d + interval 5 second SETTINGS merge_with_ttl_timeout=5""".replace('\r', '').replace('\n', '') create_mv_table2 = """ create table t2 on cluster default (a Int8) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by tuple() order by a""".replace('\r', '').replace('\n', '') create_mv_table3 = """ create table t3 on cluster default (a Int8) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by tuple() order by a""".replace('\r', '').replace('\n', '') create_mv2 = "create materialized view t_mv2 on cluster default to t2 as select a from t1" create_mv3 = "create materialized view t_mv3 on cluster default to t3 as select a from t1" with Given("Tables t1, t2, t3 and MVs t1->t2, t1-t3 are created"): clickhouse.query(chi, create_table) clickhouse.query(chi, create_mv_table2) clickhouse.query(chi, create_mv_table3) clickhouse.query(chi, create_mv2) clickhouse.query(chi, create_mv3) with When("Add a row to an old partition"): clickhouse.query(chi, "insert into t1(a,d) values(6, today()-1)", host=host0) with When("Stop fetches for t1 at replica1"): clickhouse.query(chi, "system stop fetches default.t1", host=host1) with Then("Wait 10 seconds and the data should be dropped by TTL"): time.sleep(10) out = clickhouse.query(chi, "select count() from t1 where a=6", host=host0) assert out == "0", error() with When("Resume fetches for t1 at replica1"): clickhouse.query(chi, "system start fetches default.t1", host=host1) time.sleep(5) with Then("Inserts should resume"): clickhouse.query(chi, "insert into t1(a) values(7)", host=host0) clickhouse.query(chi, "insert into t1(a) values(1)") with When("Stop fetches for t2 at replica1"): clickhouse.query(chi, "system stop fetches default.t2", host=host1) with Then("Insert should fail since it can not reach the quorum"): out = clickhouse.query_with_error( chi, "insert into t1(a) values(2)", host=host0) assert "Timeout while waiting for quorum" in out, error() # kubectl(f"exec {host0}-0 -n test -- cp /var/lib//clickhouse/data/default/t2/all_1_1_0/a.mrk2 /var/lib//clickhouse/data/default/t2/all_1_1_0/a.bin") # with Then("Corrupt data part in t2"): # kubectl(f"exec {host0}-0 -n test -- sed -i \"s/b/c/\" /var/lib/clickhouse/data/default/t2/all_1_1_0/a.bin") with When("Resume fetches for t2 at replica1"): clickhouse.query(chi, "system start fetches default.t2", host=host1) i = 0 while "2" != clickhouse.query( chi, "select active_replicas from system.replicas where database='default' and table='t1'", pod=host0) and i < 10: with Then("Not ready, wait 5 seconds"): time.sleep(5) i += 1 with Then( "Inserts should fail with an error regarding not satisfied quorum" ): out = clickhouse.query_with_error( chi, "insert into t1(a) values(3)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out, error( ) with And("Second insert of the same block should pass"): clickhouse.query(chi, "insert into t1(a) values(3)", host=host0) with And("Insert of the new block should fail"): out = clickhouse.query_with_error( chi, "insert into t1(a) values(4)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out, error( ) with And( "Second insert of the same block with 'deduplicate_blocks_in_dependent_materialized_views' setting should fail" ): out = clickhouse.query_with_error( chi, "set deduplicate_blocks_in_dependent_materialized_views=1; insert into t1(a) values(5)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out, error( ) out = clickhouse.query_with_error( chi, "select t1.a t1_a, t2.a t2_a from t1 left outer join t2 using (a) order by t1_a settings join_use_nulls=1" ) note(out)
def test_metrics_exporter_reboot(self): def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_retries=10): with Then( f"metrics-exporter /chi endpoint result should return {expect_result}" ): for i in range(1, max_retries): # check /metrics for try to refresh monitored instances url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics") kubectl.launch( f"exec {operator_pod} -c metrics-exporter -- {url_cmd}", ns=operator_namespace) # check /chi after refresh monitored instances url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/chi") out = kubectl.launch( f"exec {operator_pod} -c metrics-exporter -- {url_cmd}", ns=operator_namespace) out = json.loads(out) if out == expect_result: break with Then("Not ready. Wait for " + str(i * 5) + " seconds"): time.sleep(i * 5) assert out == expect_result, error() with Given("clickhouse-operator is installed"): kubectl.wait_field("pods", util.operator_label, ".status.containerStatuses[*].ready", "true,true", ns=settings.operator_namespace) assert kubectl.get_count("pod", ns='--all-namespaces', label=util.operator_label) > 0, error() out = kubectl.launch("get pods -l app=clickhouse-operator", ns=settings.operator_namespace).splitlines()[1] operator_pod = re.split(r'[\t\r\n\s]+', out)[0] operator_namespace = settings.operator_namespace kubectl.delete_ns(kubectl.namespace, ok_to_fail=True) kubectl.create_ns(kubectl.namespace) check_monitoring_chi(operator_namespace, operator_pod, []) with And("created simple clickhouse installation"): manifest = "../../docs/chi-examples/01-simple-layout-01-1shard-1repl.yaml" kubectl.create_and_check(manifest=manifest, check={ "object_counts": { "statefulset": 1, "pod": 1, "service": 2, }, "do_not_delete": True, }) expected_chi = [{ "namespace": "test", "name": "simple-01", "hostnames": ["chi-simple-01-simple-0-0.test.svc.cluster.local"] }] check_monitoring_chi(operator_namespace, operator_pod, expected_chi) with When("reboot metrics exporter"): kubectl.launch( f"exec -n {operator_namespace} {operator_pod} -c metrics-exporter -- bash -c 'kill 1'" ) time.sleep(15) kubectl.wait_field("pods", util.operator_label, ".status.containerStatuses[*].ready", "true,true", ns=settings.operator_namespace) with Then("check metrics exporter still contains chi objects"): check_monitoring_chi(operator_namespace, operator_pod, expected_chi) kubectl.delete(util.get_full_path(manifest, lookup_in_host=False), timeout=600) check_monitoring_chi(operator_namespace, operator_pod, [])
def test_metrics_exporter_with_multiple_clickhouse_version(self): def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, max_retries=10): with Then( f"metrics-exporter /metrics endpoint result should match with {expect_result}" ): for i in range(1, max_retries): url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics") out = kubectl.launch( f"exec {operator_pod} -c metrics-exporter -- {url_cmd}", ns=operator_namespace) all_strings_expected_done = True for string, exists in expect_result.items(): all_strings_expected_done = (exists == (string in out)) if not all_strings_expected_done: break if all_strings_expected_done: break with Then("Not ready. Wait for " + str(i * 5) + " seconds"): time.sleep(i * 5) assert all_strings_expected_done, error() with Given("clickhouse-operator pod exists"): out = kubectl.launch("get pods -l app=clickhouse-operator", ns='kube-system').splitlines()[1] operator_pod = re.split(r'[\t\r\n\s]+', out)[0] operator_namespace = "kube-system" with Then("check empty /metrics"): kubectl.delete_ns(kubectl.namespace, ok_to_fail=True) kubectl.create_ns(kubectl.namespace) check_monitoring_metrics( operator_namespace, operator_pod, expect_result={ 'chi_clickhouse_metric_VersionInteger': False, }) with Then("Install multiple clickhouse version"): manifest = "manifests/chi/test-017-multi-version.yaml" kubectl.create_and_check(manifest=manifest, check={ "object_counts": { "statefulset": 2, "pod": 2, "service": 3, }, "do_not_delete": True, }) with And("Check not empty /metrics"): check_monitoring_metrics( operator_namespace, operator_pod, expect_result={ '# HELP chi_clickhouse_metric_VersionInteger': True, '# TYPE chi_clickhouse_metric_VersionInteger gauge': True, 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True, 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True, }) with Then("check empty /metrics after delete namespace"): kubectl.delete_ns(kubectl.namespace) check_monitoring_metrics( operator_namespace, operator_pod, expect_result={ 'chi_clickhouse_metric_VersionInteger': False, })