def test_018():
kubectl.create_and_check(config="configs/test-018-configmap.yaml",
check={
"pod_count": 1,
"do_not_delete": 1,
})
chi_name = "test-018-configmap"
with Then("user1/networks/ip should be in config"):
chi = kubectl.get("chi", chi_name)
assert "user1/networks/ip" in chi["spec"]["configuration"]["users"]
start_time = kubectl.get_field("pod", f"chi-{chi_name}-default-0-0-0",
".status.startTime")
kubectl.create_and_check(config="configs/test-018-configmap-2.yaml",
check={
"pod_count": 1,
"do_not_delete": 1,
})
with Then("user2/networks should be in config"):
chi = kubectl.get("chi", chi_name)
assert "user2/networks/ip" in chi["spec"]["configuration"]["users"]
with And("user1/networks/ip should NOT be in config"):
assert "user1/networks/ip" not in chi["spec"]["configuration"][
"users"]
with And("Pod should not be restarted"):
new_start_time = kubectl.get_field(
"pod", f"chi-{chi_name}-default-0-0-0", ".status.startTime")
assert start_time == new_start_time
kubectl.delete_chi(chi_name)
def get_prometheus_and_alertmanager_spec():
with Given("get information about prometheus installation"):
prometheus_operator_spec = kubectl.get(
"pod",
ns=settings.prometheus_namespace,
name="",
label=
"-l app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator"
)
alertmanager_spec = kubectl.get(
"pod",
ns=settings.prometheus_namespace,
name="",
label="-l app=alertmanager,alertmanager=alertmanager")
prometheus_spec = kubectl.get(
"pod",
ns=settings.prometheus_namespace,
name="",
label="-l app=prometheus,prometheus=prometheus")
if not ("items" in prometheus_spec and len(prometheus_spec["items"])
and "metadata" in prometheus_spec["items"][0]):
fail("invalid prometheus_spec, please run create-prometheus.sh")
return prometheus_operator_spec, prometheus_spec, alertmanager_spec
def install_clickhouse_and_zookeeper(chi_file, chi_template_file, chi_name):
with Given("install zookeeper+clickhouse"):
kubectl.delete_ns(settings.test_namespace, ok_to_fail=True, timeout=600)
kubectl.create_ns(settings.test_namespace)
util.require_zookeeper()
kubectl.create_and_check(
config=chi_file,
check={
"apply_templates": [
chi_template_file,
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": {
"statefulset": 2,
"pod": 2,
"service": 3,
},
"do_not_delete": 1
}
)
clickhouse_operator_spec = kubectl.get(
"pod", name="", ns=settings.operator_namespace, label="-l app=clickhouse-operator"
)
chi = kubectl.get("chi", ns=settings.test_namespace, name=chi_name)
return clickhouse_operator_spec, chi
def test_011_1():
with Given(
"test-011-secured-default.yaml with password_sha256_hex for default user"
):
kubectl.create_and_check(
config="configs/test-011-secured-default.yaml",
check={
"pod_count": 1,
"do_not_delete": 1,
})
with Then("Default user password should be '_removed_'"):
chi = kubectl.get("chi", "test-011-secured-default")
assert "default/password" in chi["status"]["normalized"][
"configuration"]["users"]
assert chi["status"]["normalized"]["configuration"]["users"][
"default/password"] == "_removed_"
with And("Connection to localhost should succeed with default user"):
out = clickhouse.query_with_error(
"test-011-secured-default",
"select 'OK'",
pwd="clickhouse_operator_password")
assert out == 'OK'
with When("Trigger installation update"):
kubectl.create_and_check(
config="configs/test-011-secured-default-2.yaml",
check={
"do_not_delete": 1,
})
with Then("Default user password should be '_removed_'"):
chi = kubectl.get("chi", "test-011-secured-default")
assert "default/password" in chi["status"]["normalized"][
"configuration"]["users"]
assert chi["status"]["normalized"]["configuration"]["users"][
"default/password"] == "_removed_"
with When("Default user is assigned the different profile"):
kubectl.create_and_check(
config="configs/test-011-secured-default-3.yaml",
check={
"do_not_delete": 1,
})
with Then("Wait until configmap is reloaded"):
# Need to wait to make sure configuration is reloaded. For some reason it takes long here
# Maybe we can restart the pod to speed it up
time.sleep(120)
with Then(
"Connection to localhost should succeed with default user"
):
out = clickhouse.query_with_error("test-011-secured-default",
"select 'OK'")
assert out == 'OK'
kubectl.delete_chi("test-011-secured-default")
def restart_operator(ns=settings.operator_namespace, timeout=60):
pod_name = kubectl.get(
"pod", name="", ns=ns,
label="-l app=clickhouse-operator")["items"][0]["metadata"]["name"]
kubectl.launch(f"delete pod {pod_name}", ns=ns, timeout=timeout)
kubectl.wait_object("pod",
name="",
ns=ns,
label="-l app=clickhouse-operator")
pod_name = kubectl.get(
"pod", name="", ns=ns,
label="-l app=clickhouse-operator")["items"][0]["metadata"]["name"]
kubectl.wait_pod_status(pod_name, "Running", ns=ns)
def test_zookeeper_alerts(self):
zookeeper_spec = kubectl.get("endpoints", "zookeeper")
zookeeper_pod = random.choice(
zookeeper_spec["subsets"][0]["addresses"])["targetRef"]["name"]
def restart_zookeeper():
kubectl.launch(
f"exec -n {kubectl.namespace} {zookeeper_pod} -- sh -c \"kill 1\"",
ok_to_fail=True,
)
def wait_when_zookeeper_up():
kubectl.wait_pod_status(zookeeper_pod, "Running", ns=kubectl.namespace)
kubectl.wait_jsonpath("pod",
zookeeper_pod,
"{.status.containerStatuses[0].ready}",
"true",
ns=kubectl.namespace)
with Then("check ZookeeperDown firing"):
fired = alerts.wait_alert_state(
"ZookeeperDown",
"firing",
True,
labels={"pod_name": zookeeper_pod},
time_range='1m',
sleep_time=settings.prometheus_scrape_interval,
callback=restart_zookeeper)
assert fired, error("can't get ZookeeperDown alert in firing state")
wait_when_zookeeper_up()
with Then("check ZookeeperDown gone away"):
resolved = alerts.wait_alert_state("ZookeeperDown",
"firing",
False,
labels={"pod_name": zookeeper_pod})
assert resolved, error("can't check ZookeeperDown alert is gone away")
restart_zookeeper()
with Then("check ZookeeperRestartRecently firing"):
fired = alerts.wait_alert_state("ZookeeperRestartRecently",
"firing",
True,
labels={"pod_name": zookeeper_pod},
time_range='30s')
assert fired, error(
"can't get ZookeeperRestartRecently alert in firing state")
wait_when_zookeeper_up()
with Then("check ZookeeperRestartRecently gone away"):
resolved = alerts.wait_alert_state("ZookeeperRestartRecently",
"firing",
False,
labels={"pod_name": zookeeper_pod})
assert resolved, error(
"can't check ZookeeperRestartRecently alert is gone away")
def get_minio_spec():
with Given("get information about prometheus installation"):
minio_spec = kubectl.get("pod",
ns=settings.minio_namespace,
name="",
label="-l app=minio")
assert "items" in minio_spec and len(
minio_spec["items"]) and "metadata" in minio_spec["items"][
0], error("invalid minio spec, please run install-minio.sh")
return minio_spec
def test_012():
kubectl.create_and_check(
config="configs/test-012-service-template.yaml",
check={
"object_counts": {
"statefulset": 2,
"pod": 2,
"service": 4,
},
"do_not_delete": 1,
}
)
with Then("There should be a service for chi"):
kubectl.check_service("service-test-012", "LoadBalancer")
with And("There should be a service for shard 0"):
kubectl.check_service("service-test-012-0-0", "ClusterIP")
with And("There should be a service for shard 1"):
kubectl.check_service("service-test-012-1-0", "ClusterIP")
with And("There should be a service for default cluster"):
kubectl.check_service("service-default", "ClusterIP")
node_port = kubectl.get("service", "service-test-012")["spec"]["ports"][0]["nodePort"]
with Then("Update chi"):
kubectl.create_and_check(
config="configs/test-012-service-template-2.yaml",
check={
"object_counts": {
"statefulset": 1,
"pod": 1,
"service": 3,
},
"do_not_delete": 1,
}
)
with And("NodePort should not change"):
new_node_port = kubectl.get("service", "service-test-012")["spec"]["ports"][0]["nodePort"]
assert new_node_port == node_port, \
f"LoadBalancer.spec.ports[0].nodePort changed from {node_port} to {new_node_port}"
kubectl.delete_chi("test-012")
def test_ch_001(self):
util.require_zookeeper()
chit_data = manifest.get_chit_data(
util.get_full_path("templates/tpl-clickhouse-19.11.yaml"))
kubectl.launch(f"delete chit {chit_data['metadata']['name']}",
ns=settings.test_namespace)
kubectl.create_and_check(
"configs/test-ch-001-insert-quorum.yaml", {
"apply_templates": {"templates/tpl-clickhouse-20.8.yaml"},
"pod_count": 2,
"do_not_delete": 1,
})
chi = manifest.get_chi_name(
util.get_full_path("configs/test-ch-001-insert-quorum.yaml"))
chi_data = kubectl.get("chi", ns=settings.test_namespace, name=chi)
util.wait_clickhouse_cluster_ready(chi_data)
host0 = "chi-test-ch-001-insert-quorum-default-0-0"
host1 = "chi-test-ch-001-insert-quorum-default-0-1"
create_table = """
create table t1 on cluster default (a Int8, d Date default today())
Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}')
partition by d order by a
TTL d + interval 5 second
SETTINGS merge_with_ttl_timeout=5""".replace('\r', '').replace('\n', '')
create_mv_table2 = """
create table t2 on cluster default (a Int8)
Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}')
partition by tuple() order by a""".replace('\r', '').replace('\n', '')
create_mv_table3 = """
create table t3 on cluster default (a Int8)
Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}')
partition by tuple() order by a""".replace('\r', '').replace('\n', '')
create_mv2 = "create materialized view t_mv2 on cluster default to t2 as select a from t1"
create_mv3 = "create materialized view t_mv3 on cluster default to t3 as select a from t1"
with Given("Tables t1, t2, t3 and MVs t1->t2, t1-t3 are created"):
clickhouse.query(chi, create_table)
clickhouse.query(chi, create_mv_table2)
clickhouse.query(chi, create_mv_table3)
clickhouse.query(chi, create_mv2)
clickhouse.query(chi, create_mv3)
with When("Add a row to an old partition"):
clickhouse.query(chi,
"insert into t1(a,d) values(6, today()-1)",
host=host0)
with When("Stop fetches for t1 at replica1"):
clickhouse.query(chi, "system stop fetches default.t1", host=host1)
with Then("Wait 10 seconds and the data should be dropped by TTL"):
time.sleep(10)
out = clickhouse.query(chi,
"select count() from t1 where a=6",
host=host0)
assert out == "0"
with When("Resume fetches for t1 at replica1"):
clickhouse.query(chi,
"system start fetches default.t1",
host=host1)
time.sleep(5)
with Then("Inserts should resume"):
clickhouse.query(chi,
"insert into t1(a) values(7)",
host=host0)
clickhouse.query(chi, "insert into t1(a) values(1)")
with When("Stop fetches for t2 at replica1"):
clickhouse.query(chi, "system stop fetches default.t2", host=host1)
with Then("Insert should fail since it can not reach the quorum"):
out = clickhouse.query_with_error(
chi, "insert into t1(a) values(2)", host=host0)
assert "Timeout while waiting for quorum" in out
# kubectl(f"exec {host0}-0 -n test -- cp /var/lib//clickhouse/data/default/t2/all_1_1_0/a.mrk2 /var/lib//clickhouse/data/default/t2/all_1_1_0/a.bin")
# with Then("Corrupt data part in t2"):
# kubectl(f"exec {host0}-0 -n test -- sed -i \"s/b/c/\" /var/lib/clickhouse/data/default/t2/all_1_1_0/a.bin")
with When("Resume fetches for t2 at replica1"):
clickhouse.query(chi,
"system start fetches default.t2",
host=host1)
i = 0
while "2" != clickhouse.query(
chi,
"select active_replicas from system.replicas where database='default' and table='t1'",
pod=host0) and i < 10:
with Then("Not ready, wait 5 seconds"):
time.sleep(5)
i += 1
with Then(
"Inserts should fail with an error regarding not satisfied quorum"
):
out = clickhouse.query_with_error(
chi, "insert into t1(a) values(3)", host=host0)
assert "Quorum for previous write has not been satisfied yet" in out
with And("Second insert of the same block should pass"):
clickhouse.query(chi,
"insert into t1(a) values(3)",
host=host0)
with And("Insert of the new block should fail"):
out = clickhouse.query_with_error(
chi, "insert into t1(a) values(4)", host=host0)
assert "Quorum for previous write has not been satisfied yet" in out
with And(
"Second insert of the same block with 'deduplicate_blocks_in_dependent_materialized_views' setting should fail"
):
out = clickhouse.query_with_error(
chi,
"set deduplicate_blocks_in_dependent_materialized_views=1; insert into t1(a) values(5)",
host=host0)
assert "Quorum for previous write has not been satisfied yet" in out
out = clickhouse.query_with_error(
chi,
"select t1.a t1_a, t2.a t2_a from t1 left outer join t2 using (a) order by t1_a settings join_use_nulls=1"
)
print(out)
fired = wait_alert_state("ZookeeperRestartRecently", "firing", True, labels={"pod": zookeeper_pod},
time_range='30s', sleep_time=5)
assert fired, error("can't get ZookeeperRestartRecently alert in firing state")
wait_when_zookeeper_up()
with Then("check ZookeeperRestartRecently gone away"):
resolved = wait_alert_state("ZookeeperRestartRecently", "firing", False, labels={"pod": zookeeper_pod})
assert resolved, error("can't check ZookeeperRestartRecently alert is gone away")
if main():
with Module("main"):
with Given("get information about prometheus installation"):
prometheus_operator_spec = kubectl.get(
"pod", ns=settings.prometheus_namespace, name="",
label="-l app.kubernetes.io/component=controller,app.kubernetes.io/name=prometheus-operator"
)
alertmanager_spec = kubectl.get(
"pod", ns=settings.prometheus_namespace, name="",
label="-l app=alertmanager,alertmanager=alertmanager"
)
prometheus_spec = kubectl.get(
"pod", ns=settings.prometheus_namespace, name="",
label="-l app=prometheus,prometheus=prometheus"
)
assert "items" in prometheus_spec and len(prometheus_spec["items"]) > 0 and "metadata" in prometheus_spec["items"][0], "invalid prometheus_spec"
with Given("install zookeeper+clickhouse"):
kubectl.delete_ns(kubectl.namespace, ok_to_fail=True)
def test_get(self):
self.assertEquals(
kubectl.get('namespaces', binary=TEST_BINARY),
'kubectl get namespaces --output=json')
def test_011():
with Given(
"test-011-secured-cluster.yaml and test-011-insecured-cluster.yaml"
):
kubectl.create_and_check(
config="configs/test-011-secured-cluster.yaml",
check={
"pod_count":
2,
"service": [
"chi-test-011-secured-cluster-default-1-0",
"ClusterIP",
],
"apply_templates": {
settings.clickhouse_template,
"templates/tpl-log-volume.yaml",
},
"do_not_delete":
1,
})
kubectl.create_and_check(
config="configs/test-011-insecured-cluster.yaml",
check={
"pod_count": 1,
"do_not_delete": 1,
})
time.sleep(60)
with Then("Connection to localhost should succeed with default user"):
out = clickhouse.query_with_error("test-011-secured-cluster",
"select 'OK'")
assert out == 'OK', f"out={out} should be 'OK'"
with And("Connection from secured to secured host should succeed"):
out = clickhouse.query_with_error(
"test-011-secured-cluster",
"select 'OK'",
host="chi-test-011-secured-cluster-default-1-0")
assert out == 'OK'
with And(
"Connection from insecured to secured host should fail for default"
):
out = clickhouse.query_with_error(
"test-011-insecured-cluster",
"select 'OK'",
host="chi-test-011-secured-cluster-default-1-0")
assert out != 'OK'
with And(
"Connection from insecured to secured host should fail for user with no password"
):
time.sleep(10) # FIXME
out = clickhouse.query_with_error(
"test-011-insecured-cluster",
"select 'OK'",
host="chi-test-011-secured-cluster-default-1-0",
user="******")
assert "Password" in out or "password" in out
with And(
"Connection from insecured to secured host should work for user with password"
):
out = clickhouse.query_with_error(
"test-011-insecured-cluster",
"select 'OK'",
host="chi-test-011-secured-cluster-default-1-0",
user="******",
pwd="topsecret")
assert out == 'OK'
with And("Password should be encrypted"):
cfm = kubectl.get("configmap",
"chi-test-011-secured-cluster-common-usersd")
users_xml = cfm["data"]["chop-generated-users.xml"]
assert "<password>" not in users_xml
assert "<password_sha256_hex>" in users_xml
with And("User with no password should get default automatically"):
out = clickhouse.query_with_error("test-011-secured-cluster",
"select 'OK'",
user="******",
pwd="default")
assert out == 'OK'
with And(
"User with both plain and sha256 password should get the latter one"
):
out = clickhouse.query_with_error(
"test-011-secured-cluster",
"select 'OK'",
user="******",
pwd="clickhouse_operator_password")
assert out == 'OK'
with And("User with row-level security should have it applied"):
out = clickhouse.query_with_error(
"test-011-secured-cluster",
"select * from system.numbers limit 1",
user="******",
pwd="secret")
assert out == '1000'
kubectl.delete_chi("test-011-secured-cluster")
kubectl.delete_chi("test-011-insecured-cluster")
