def test_018(): kubectl.create_and_check(config="configs/test-018-configmap.yaml", check={ "pod_count": 1, "do_not_delete": 1, }) chi_name = "test-018-configmap" with Then("user1/networks/ip should be in config"): chi = kubectl.get("chi", chi_name) assert "user1/networks/ip" in chi["spec"]["configuration"]["users"] start_time = kubectl.get_field("pod", f"chi-{chi_name}-default-0-0-0", ".status.startTime") kubectl.create_and_check(config="configs/test-018-configmap-2.yaml", check={ "pod_count": 1, "do_not_delete": 1, }) with Then("user2/networks should be in config"): chi = kubectl.get("chi", chi_name) assert "user2/networks/ip" in chi["spec"]["configuration"]["users"] with And("user1/networks/ip should NOT be in config"): assert "user1/networks/ip" not in chi["spec"]["configuration"][ "users"] with And("Pod should not be restarted"): new_start_time = kubectl.get_field( "pod", f"chi-{chi_name}-default-0-0-0", ".status.startTime") assert start_time == new_start_time kubectl.delete_chi(chi_name)
def get_prometheus_and_alertmanager_spec(): with Given("get information about prometheus installation"): prometheus_operator_spec = kubectl.get( "pod", ns=settings.prometheus_namespace, name="", label= "-l app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator" ) alertmanager_spec = kubectl.get( "pod", ns=settings.prometheus_namespace, name="", label="-l app=alertmanager,alertmanager=alertmanager") prometheus_spec = kubectl.get( "pod", ns=settings.prometheus_namespace, name="", label="-l app=prometheus,prometheus=prometheus") if not ("items" in prometheus_spec and len(prometheus_spec["items"]) and "metadata" in prometheus_spec["items"][0]): fail("invalid prometheus_spec, please run create-prometheus.sh") return prometheus_operator_spec, prometheus_spec, alertmanager_spec
def install_clickhouse_and_zookeeper(chi_file, chi_template_file, chi_name): with Given("install zookeeper+clickhouse"): kubectl.delete_ns(settings.test_namespace, ok_to_fail=True, timeout=600) kubectl.create_ns(settings.test_namespace) util.require_zookeeper() kubectl.create_and_check( config=chi_file, check={ "apply_templates": [ chi_template_file, "templates/tpl-persistent-volume-100Mi.yaml" ], "object_counts": { "statefulset": 2, "pod": 2, "service": 3, }, "do_not_delete": 1 } ) clickhouse_operator_spec = kubectl.get( "pod", name="", ns=settings.operator_namespace, label="-l app=clickhouse-operator" ) chi = kubectl.get("chi", ns=settings.test_namespace, name=chi_name) return clickhouse_operator_spec, chi
def test_011_1(): with Given( "test-011-secured-default.yaml with password_sha256_hex for default user" ): kubectl.create_and_check( config="configs/test-011-secured-default.yaml", check={ "pod_count": 1, "do_not_delete": 1, }) with Then("Default user password should be '_removed_'"): chi = kubectl.get("chi", "test-011-secured-default") assert "default/password" in chi["status"]["normalized"][ "configuration"]["users"] assert chi["status"]["normalized"]["configuration"]["users"][ "default/password"] == "_removed_" with And("Connection to localhost should succeed with default user"): out = clickhouse.query_with_error( "test-011-secured-default", "select 'OK'", pwd="clickhouse_operator_password") assert out == 'OK' with When("Trigger installation update"): kubectl.create_and_check( config="configs/test-011-secured-default-2.yaml", check={ "do_not_delete": 1, }) with Then("Default user password should be '_removed_'"): chi = kubectl.get("chi", "test-011-secured-default") assert "default/password" in chi["status"]["normalized"][ "configuration"]["users"] assert chi["status"]["normalized"]["configuration"]["users"][ "default/password"] == "_removed_" with When("Default user is assigned the different profile"): kubectl.create_and_check( config="configs/test-011-secured-default-3.yaml", check={ "do_not_delete": 1, }) with Then("Wait until configmap is reloaded"): # Need to wait to make sure configuration is reloaded. For some reason it takes long here # Maybe we can restart the pod to speed it up time.sleep(120) with Then( "Connection to localhost should succeed with default user" ): out = clickhouse.query_with_error("test-011-secured-default", "select 'OK'") assert out == 'OK' kubectl.delete_chi("test-011-secured-default")
def restart_operator(ns=settings.operator_namespace, timeout=60): pod_name = kubectl.get( "pod", name="", ns=ns, label="-l app=clickhouse-operator")["items"][0]["metadata"]["name"] kubectl.launch(f"delete pod {pod_name}", ns=ns, timeout=timeout) kubectl.wait_object("pod", name="", ns=ns, label="-l app=clickhouse-operator") pod_name = kubectl.get( "pod", name="", ns=ns, label="-l app=clickhouse-operator")["items"][0]["metadata"]["name"] kubectl.wait_pod_status(pod_name, "Running", ns=ns)
def test_zookeeper_alerts(self): zookeeper_spec = kubectl.get("endpoints", "zookeeper") zookeeper_pod = random.choice( zookeeper_spec["subsets"][0]["addresses"])["targetRef"]["name"] def restart_zookeeper(): kubectl.launch( f"exec -n {kubectl.namespace} {zookeeper_pod} -- sh -c \"kill 1\"", ok_to_fail=True, ) def wait_when_zookeeper_up(): kubectl.wait_pod_status(zookeeper_pod, "Running", ns=kubectl.namespace) kubectl.wait_jsonpath("pod", zookeeper_pod, "{.status.containerStatuses[0].ready}", "true", ns=kubectl.namespace) with Then("check ZookeeperDown firing"): fired = alerts.wait_alert_state( "ZookeeperDown", "firing", True, labels={"pod_name": zookeeper_pod}, time_range='1m', sleep_time=settings.prometheus_scrape_interval, callback=restart_zookeeper) assert fired, error("can't get ZookeeperDown alert in firing state") wait_when_zookeeper_up() with Then("check ZookeeperDown gone away"): resolved = alerts.wait_alert_state("ZookeeperDown", "firing", False, labels={"pod_name": zookeeper_pod}) assert resolved, error("can't check ZookeeperDown alert is gone away") restart_zookeeper() with Then("check ZookeeperRestartRecently firing"): fired = alerts.wait_alert_state("ZookeeperRestartRecently", "firing", True, labels={"pod_name": zookeeper_pod}, time_range='30s') assert fired, error( "can't get ZookeeperRestartRecently alert in firing state") wait_when_zookeeper_up() with Then("check ZookeeperRestartRecently gone away"): resolved = alerts.wait_alert_state("ZookeeperRestartRecently", "firing", False, labels={"pod_name": zookeeper_pod}) assert resolved, error( "can't check ZookeeperRestartRecently alert is gone away")
def get_minio_spec(): with Given("get information about prometheus installation"): minio_spec = kubectl.get("pod", ns=settings.minio_namespace, name="", label="-l app=minio") assert "items" in minio_spec and len( minio_spec["items"]) and "metadata" in minio_spec["items"][ 0], error("invalid minio spec, please run install-minio.sh") return minio_spec
def test_012(): kubectl.create_and_check( config="configs/test-012-service-template.yaml", check={ "object_counts": { "statefulset": 2, "pod": 2, "service": 4, }, "do_not_delete": 1, } ) with Then("There should be a service for chi"): kubectl.check_service("service-test-012", "LoadBalancer") with And("There should be a service for shard 0"): kubectl.check_service("service-test-012-0-0", "ClusterIP") with And("There should be a service for shard 1"): kubectl.check_service("service-test-012-1-0", "ClusterIP") with And("There should be a service for default cluster"): kubectl.check_service("service-default", "ClusterIP") node_port = kubectl.get("service", "service-test-012")["spec"]["ports"][0]["nodePort"] with Then("Update chi"): kubectl.create_and_check( config="configs/test-012-service-template-2.yaml", check={ "object_counts": { "statefulset": 1, "pod": 1, "service": 3, }, "do_not_delete": 1, } ) with And("NodePort should not change"): new_node_port = kubectl.get("service", "service-test-012")["spec"]["ports"][0]["nodePort"] assert new_node_port == node_port, \ f"LoadBalancer.spec.ports[0].nodePort changed from {node_port} to {new_node_port}" kubectl.delete_chi("test-012")
def test_ch_001(self): util.require_zookeeper() chit_data = manifest.get_chit_data( util.get_full_path("templates/tpl-clickhouse-19.11.yaml")) kubectl.launch(f"delete chit {chit_data['metadata']['name']}", ns=settings.test_namespace) kubectl.create_and_check( "configs/test-ch-001-insert-quorum.yaml", { "apply_templates": {"templates/tpl-clickhouse-20.8.yaml"}, "pod_count": 2, "do_not_delete": 1, }) chi = manifest.get_chi_name( util.get_full_path("configs/test-ch-001-insert-quorum.yaml")) chi_data = kubectl.get("chi", ns=settings.test_namespace, name=chi) util.wait_clickhouse_cluster_ready(chi_data) host0 = "chi-test-ch-001-insert-quorum-default-0-0" host1 = "chi-test-ch-001-insert-quorum-default-0-1" create_table = """ create table t1 on cluster default (a Int8, d Date default today()) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by d order by a TTL d + interval 5 second SETTINGS merge_with_ttl_timeout=5""".replace('\r', '').replace('\n', '') create_mv_table2 = """ create table t2 on cluster default (a Int8) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by tuple() order by a""".replace('\r', '').replace('\n', '') create_mv_table3 = """ create table t3 on cluster default (a Int8) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by tuple() order by a""".replace('\r', '').replace('\n', '') create_mv2 = "create materialized view t_mv2 on cluster default to t2 as select a from t1" create_mv3 = "create materialized view t_mv3 on cluster default to t3 as select a from t1" with Given("Tables t1, t2, t3 and MVs t1->t2, t1-t3 are created"): clickhouse.query(chi, create_table) clickhouse.query(chi, create_mv_table2) clickhouse.query(chi, create_mv_table3) clickhouse.query(chi, create_mv2) clickhouse.query(chi, create_mv3) with When("Add a row to an old partition"): clickhouse.query(chi, "insert into t1(a,d) values(6, today()-1)", host=host0) with When("Stop fetches for t1 at replica1"): clickhouse.query(chi, "system stop fetches default.t1", host=host1) with Then("Wait 10 seconds and the data should be dropped by TTL"): time.sleep(10) out = clickhouse.query(chi, "select count() from t1 where a=6", host=host0) assert out == "0" with When("Resume fetches for t1 at replica1"): clickhouse.query(chi, "system start fetches default.t1", host=host1) time.sleep(5) with Then("Inserts should resume"): clickhouse.query(chi, "insert into t1(a) values(7)", host=host0) clickhouse.query(chi, "insert into t1(a) values(1)") with When("Stop fetches for t2 at replica1"): clickhouse.query(chi, "system stop fetches default.t2", host=host1) with Then("Insert should fail since it can not reach the quorum"): out = clickhouse.query_with_error( chi, "insert into t1(a) values(2)", host=host0) assert "Timeout while waiting for quorum" in out # kubectl(f"exec {host0}-0 -n test -- cp /var/lib//clickhouse/data/default/t2/all_1_1_0/a.mrk2 /var/lib//clickhouse/data/default/t2/all_1_1_0/a.bin") # with Then("Corrupt data part in t2"): # kubectl(f"exec {host0}-0 -n test -- sed -i \"s/b/c/\" /var/lib/clickhouse/data/default/t2/all_1_1_0/a.bin") with When("Resume fetches for t2 at replica1"): clickhouse.query(chi, "system start fetches default.t2", host=host1) i = 0 while "2" != clickhouse.query( chi, "select active_replicas from system.replicas where database='default' and table='t1'", pod=host0) and i < 10: with Then("Not ready, wait 5 seconds"): time.sleep(5) i += 1 with Then( "Inserts should fail with an error regarding not satisfied quorum" ): out = clickhouse.query_with_error( chi, "insert into t1(a) values(3)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out with And("Second insert of the same block should pass"): clickhouse.query(chi, "insert into t1(a) values(3)", host=host0) with And("Insert of the new block should fail"): out = clickhouse.query_with_error( chi, "insert into t1(a) values(4)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out with And( "Second insert of the same block with 'deduplicate_blocks_in_dependent_materialized_views' setting should fail" ): out = clickhouse.query_with_error( chi, "set deduplicate_blocks_in_dependent_materialized_views=1; insert into t1(a) values(5)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out out = clickhouse.query_with_error( chi, "select t1.a t1_a, t2.a t2_a from t1 left outer join t2 using (a) order by t1_a settings join_use_nulls=1" ) print(out)
fired = wait_alert_state("ZookeeperRestartRecently", "firing", True, labels={"pod": zookeeper_pod}, time_range='30s', sleep_time=5) assert fired, error("can't get ZookeeperRestartRecently alert in firing state") wait_when_zookeeper_up() with Then("check ZookeeperRestartRecently gone away"): resolved = wait_alert_state("ZookeeperRestartRecently", "firing", False, labels={"pod": zookeeper_pod}) assert resolved, error("can't check ZookeeperRestartRecently alert is gone away") if main(): with Module("main"): with Given("get information about prometheus installation"): prometheus_operator_spec = kubectl.get( "pod", ns=settings.prometheus_namespace, name="", label="-l app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator" ) alertmanager_spec = kubectl.get( "pod", ns=settings.prometheus_namespace, name="", label="-l app=alertmanager,alertmanager=alertmanager" ) prometheus_spec = kubectl.get( "pod", ns=settings.prometheus_namespace, name="", label="-l app=prometheus,prometheus=prometheus" ) assert "items" in prometheus_spec and len(prometheus_spec["items"]) > 0 and "metadata" in prometheus_spec["items"][0], "invalid prometheus_spec" with Given("install zookeeper+clickhouse"): kubectl.delete_ns(kubectl.namespace, ok_to_fail=True)
def test_get(self): self.assertEquals( kubectl.get('namespaces', binary=TEST_BINARY), 'kubectl get namespaces --output=json')
def test_011(): with Given( "test-011-secured-cluster.yaml and test-011-insecured-cluster.yaml" ): kubectl.create_and_check( config="configs/test-011-secured-cluster.yaml", check={ "pod_count": 2, "service": [ "chi-test-011-secured-cluster-default-1-0", "ClusterIP", ], "apply_templates": { settings.clickhouse_template, "templates/tpl-log-volume.yaml", }, "do_not_delete": 1, }) kubectl.create_and_check( config="configs/test-011-insecured-cluster.yaml", check={ "pod_count": 1, "do_not_delete": 1, }) time.sleep(60) with Then("Connection to localhost should succeed with default user"): out = clickhouse.query_with_error("test-011-secured-cluster", "select 'OK'") assert out == 'OK', f"out={out} should be 'OK'" with And("Connection from secured to secured host should succeed"): out = clickhouse.query_with_error( "test-011-secured-cluster", "select 'OK'", host="chi-test-011-secured-cluster-default-1-0") assert out == 'OK' with And( "Connection from insecured to secured host should fail for default" ): out = clickhouse.query_with_error( "test-011-insecured-cluster", "select 'OK'", host="chi-test-011-secured-cluster-default-1-0") assert out != 'OK' with And( "Connection from insecured to secured host should fail for user with no password" ): time.sleep(10) # FIXME out = clickhouse.query_with_error( "test-011-insecured-cluster", "select 'OK'", host="chi-test-011-secured-cluster-default-1-0", user="******") assert "Password" in out or "password" in out with And( "Connection from insecured to secured host should work for user with password" ): out = clickhouse.query_with_error( "test-011-insecured-cluster", "select 'OK'", host="chi-test-011-secured-cluster-default-1-0", user="******", pwd="topsecret") assert out == 'OK' with And("Password should be encrypted"): cfm = kubectl.get("configmap", "chi-test-011-secured-cluster-common-usersd") users_xml = cfm["data"]["chop-generated-users.xml"] assert "<password>" not in users_xml assert "<password_sha256_hex>" in users_xml with And("User with no password should get default automatically"): out = clickhouse.query_with_error("test-011-secured-cluster", "select 'OK'", user="******", pwd="default") assert out == 'OK' with And( "User with both plain and sha256 password should get the latter one" ): out = clickhouse.query_with_error( "test-011-secured-cluster", "select 'OK'", user="******", pwd="clickhouse_operator_password") assert out == 'OK' with And("User with row-level security should have it applied"): out = clickhouse.query_with_error( "test-011-secured-cluster", "select * from system.numbers limit 1", user="******", pwd="secret") assert out == '1000' kubectl.delete_chi("test-011-secured-cluster") kubectl.delete_chi("test-011-insecured-cluster")