def test_version_changed():
changed_pod, changed_svc, _, _ = random_pod_choice_for_callbacks()
with When("apply changed settings"):
create_and_check(
"configs/test-cluster-for-alerts-changed-settings.yaml",
{
"apply_templates": [
"templates/tpl-clickhouse-20.3.yaml",
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": [2, 2, 3],
"do_not_delete": 1
}
)
prometheus_scrape_interval = 30
with And(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 sec"):
time.sleep(prometheus_scrape_interval * 2)
with Then("check ClickHouseVersionChanged firing"):
fired = wait_alert_state("ClickHouseVersionChanged", "firing", True, labels={"hostname": changed_svc},
time_range="30s", sleep_time=5)
assert fired, error("can't get ClickHouseVersionChanged alert in firing state")
with When("rollback changed settings"):
create_and_check(
"configs/test-cluster-for-alerts.yaml",
{
"apply_templates": [
"templates/tpl-clickhouse-latest.yaml",
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": [2, 2, 3],
"do_not_delete": 1
}
)
with Then("check ClickHouseVersionChanged gone away"):
resolved = wait_alert_state("ClickHouseVersionChanged", "firing", False, labels={"hostname": changed_svc}, sleep_time=30)
assert resolved, error("can't check ClickHouseVersionChanged alert is gone away")
def test_system_settings_changed():
changed_pod, changed_svc, _, _ = random_pod_choice_for_callbacks()
with When("apply changed settings"):
create_and_check(
"configs/test-cluster-for-alerts-changed-settings.yaml",
{
"apply_templates": [
"templates/tpl-clickhouse-latest.yaml",
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": [2, 2, 3],
"do_not_delete": 1
}
)
with Then("check ClickHouseSystemSettingsChanged firing"):
fired = wait_alert_state("ClickHouseSystemSettingsChanged", "firing", True, labels={"hostname": changed_svc},
time_range="30s", sleep_time=5)
assert fired, error("can't get ClickHouseTooManyConnections alert in firing state")
with When("rollback changed settings"):
create_and_check(
"configs/test-cluster-for-alerts.yaml",
{
"apply_templates": [
"templates/tpl-clickhouse-latest.yaml",
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": [2, 2, 3],
"do_not_delete": 1
}
)
with Then("check ClickHouseSystemSettingsChanged gone away"):
resolved = wait_alert_state("ClickHouseSystemSettingsChanged", "firing", False, labels={"hostname": changed_svc}, sleep_time=30)
assert resolved, error("can't check ClickHouseTooManyConnections alert is gone away")
prometheus_spec = kubectl.kube_get(
"pod", ns=settings.prometheus_namespace, name="",
label="-l app=prometheus,prometheus=prometheus"
)
assert "items" in prometheus_spec and len(prometheus_spec["items"]) > 0 and "metadata" in prometheus_spec["items"][0], "invalid prometheus_spec"
with Given("install zookeeper+clickhouse"):
kubectl.kube_deletens(kubectl.namespace)
kubectl.kube_createns(kubectl.namespace)
require_zookeeper()
create_and_check(
"configs/test-cluster-for-alerts.yaml",
{
"apply_templates": [
"templates/tpl-clickhouse-latest.yaml",
"templates/tpl-persistent-volume-100Mi.yaml"
],
"object_counts": [2, 2, 3],
"do_not_delete": 1
}
)
clickhouse_operator_spec = kubectl.kube_get(
"pod", name="", ns=settings.operator_namespace, label="-l app=clickhouse-operator"
)
chi = kubectl.kube_get("chi", ns=kubectl.namespace, name="test-cluster-for-alerts")
with Module("metrics_alerts"):
test_cases = [
test_prometheus_setup,
test_read_only_replica,
test_metrics_exporter_down,