def test_metrics_exporter_with_multiple_clickhouse_version():
def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, max_retries=10):
with And(f"metrics-exporter /metrics enpoint result should match with {expect_result}"):
for i in range(1, max_retries):
out = kubectl.launch(
f"exec {operator_pod} -c metrics-exporter -- wget -O- -q http://127.0.0.1:8888/metrics",
ns=operator_namespace
)
all_strings_expected_done = True
for string, exists in expect_result.items():
all_strings_expected_done = (exists == (string in out))
if not all_strings_expected_done:
break
if all_strings_expected_done:
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert all_strings_expected_done, error()
with Given("clickhouse-operator pod exists"):
out = kubectl.launch("get pods -l app=clickhouse-operator", ns='kube-system').splitlines()[1]
operator_pod = re.split(r'[\t\r\n\s]+', out)[0]
operator_namespace = "kube-system"
with Then("check empty /metrics"):
kubectl.delete_ns(kubectl.namespace, ok_to_fail=True)
kubectl.create_ns(kubectl.namespace)
check_monitoring_metrics(operator_namespace, operator_pod, expect_result={
'chi_clickhouse_metric_VersionInteger': False,
})
with Then("Install multiple clickhouse version"):
config = util.get_full_path("configs/test-017-multi-version.yaml")
kubectl.create_and_check(
config=config,
check={
"object_counts": {
"statefulset": 4,
"pod": 4,
"service": 5,
},
"do_not_delete": True,
})
with And("Check not empty /metrics"):
check_monitoring_metrics(operator_namespace, operator_pod, expect_result={
'# HELP chi_clickhouse_metric_VersionInteger': True,
'# TYPE chi_clickhouse_metric_VersionInteger gauge': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-2-0': True,
'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-3-0': True,
})
with Then("check empty /metrics after delete namespace"):
kubectl.delete_ns(kubectl.namespace)
check_monitoring_metrics(operator_namespace, operator_pod, expect_result={
'chi_clickhouse_metric_VersionInteger': False,
})
def install_clickhouse_and_zookeeper(chi_file, chi_template_file, chi_name):
with Given("install zookeeper+clickhouse"):
kubectl.delete_ns(settings.test_namespace, ok_to_fail=True, timeout=600)
kubectl.create_ns(settings.test_namespace)
util.require_zookeeper()
kubectl.create_and_check(
config=chi_file,
check={
"apply_templates": [
chi_template_file,
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": {
"statefulset": 2,
"pod": 2,
"service": 3,
},
"do_not_delete": 1
}
)
clickhouse_operator_spec = kubectl.get(
"pod", name="", ns=settings.operator_namespace, label="-l app=clickhouse-operator"
)
chi = kubectl.get("chi", ns=settings.test_namespace, name=chi_name)
return clickhouse_operator_spec, chi
def test_metrics_exporter_reboot():
def check_monitoring_chi(operator_namespace, operator_pod, expect_result, max_retries=10):
with And(f"metrics-exporter /chi enpoint result should return {expect_result}"):
for i in range(1, max_retries):
# check /metrics for try to refresh monitored instances
kubectl.launch(
f"exec {operator_pod} -c metrics-exporter -- wget -O- -q http://127.0.0.1:8888/metrics",
ns=operator_namespace
)
# check /chi after refresh monitored instances
out = kubectl.launch(
f"exec {operator_pod} -c metrics-exporter -- wget -O- -q http://127.0.0.1:8888/chi",
ns=operator_namespace
)
out = json.loads(out)
if out == expect_result:
break
with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
time.sleep(i * 5)
assert out == expect_result, error()
with Given("clickhouse-operator is installed"):
kubectl.wait_field("pods", "-l app=clickhouse-operator", ".status.containerStatuses[*].ready", "true,true",
ns=settings.operator_namespace)
assert kubectl.get_count("pod", ns='--all-namespaces', label="-l app=clickhouse-operator") > 0, error()
out = kubectl.launch("get pods -l app=clickhouse-operator", ns=settings.operator_namespace).splitlines()[1]
operator_pod = re.split(r'[\t\r\n\s]+', out)[0]
operator_namespace = settings.operator_namespace
kubectl.delete_ns(kubectl.namespace)
kubectl.create_ns(kubectl.namespace)
check_monitoring_chi(operator_namespace, operator_pod, [])
with And("created simple clickhouse installation"):
config = util.get_full_path("../docs/chi-examples/01-simple-layout-01-1shard-1repl.yaml")
kubectl.create_and_check(
config=config,
check={
"object_counts": {
"statefulset": 1,
"pod": 1,
"service": 2,
},
"do_not_delete": True,
})
expected_chi = [{
"namespace": "test", "name": "simple-01",
"hostnames": ["chi-simple-01-cluster-0-0.test.svc.cluster.local"]
}]
check_monitoring_chi(operator_namespace, operator_pod, expected_chi)
with When("reboot metrics exporter"):
kubectl.launch(f"exec -n {operator_namespace} {operator_pod} -c metrics-exporter -- reboot")
time.sleep(15)
kubectl.wait_field("pods", "-l app=clickhouse-operator",
".status.containerStatuses[*].ready", "true,true",
ns=settings.operator_namespace)
with Then("check metrics exporter still contains chi objects"):
check_monitoring_chi(operator_namespace, operator_pod, expected_chi)
kubectl.delete(config)
check_monitoring_chi(operator_namespace, operator_pod, [])
)
alertmanager_spec = kubectl.get(
"pod", ns=settings.prometheus_namespace, name="",
label="-l app=alertmanager,alertmanager=alertmanager"
)
prometheus_spec = kubectl.get(
"pod", ns=settings.prometheus_namespace, name="",
label="-l app=prometheus,prometheus=prometheus"
)
assert "items" in prometheus_spec and len(prometheus_spec["items"]) > 0 and "metadata" in prometheus_spec["items"][0], "invalid prometheus_spec"
with Given("install zookeeper+clickhouse"):
kubectl.delete_ns(kubectl.namespace, ok_to_fail=True)
kubectl.create_ns(kubectl.namespace)
require_zookeeper()
kubectl.create_and_check(
config="configs/test-cluster-for-alerts.yaml",
check={
"apply_templates": [
"templates/tpl-clickhouse-latest.yaml",
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": {
"statefulset": 2,
"pod": 2,
"service": 3,
},
"do_not_delete": 1
}
import kubectl
import settings
import test_operator
import test_clickhouse
import util
from testflows.core import TestScenario, Name, When, Then, Given, And, main, run, Module, TE, args, Fail, Error
from testflows.asserts import error
if main():
with Module("main"):
with Given(f"Clean namespace {settings.test_namespace}"):
kubectl.delete_all_chi(settings.test_namespace)
kubectl.delete_ns(settings.test_namespace, ok_to_fail=True)
kubectl.create_ns(settings.test_namespace)
with Given(
f"clickhouse-operator version {settings.operator_version} is installed"
):
if kubectl.get_count("pod",
ns=settings.operator_namespace,
label="-l app=clickhouse-operator") == 0:
config = util.get_full_path(
'../deploy/operator/clickhouse-operator-install-template.yaml'
)
kubectl.apply(
ns=settings.operator_namespace,
config=f"<(cat {config} | "
f"OPERATOR_IMAGE=\"{settings.operator_docker_repo}:{settings.operator_version}\" "
f"OPERATOR_NAMESPACE=\"{settings.operator_namespace}\" "
f"METRICS_EXPORTER_IMAGE=\"{settings.metrics_exporter_docker_repo}:{settings.operator_version}\" "