def test_check_query_range_result_exp_metric_num(query_range_result_ok):
"""
Check that exp_metric_num is checked as expected when specified.
"""
result1 = check_query_range_result_enum(query_range_result_ok,
good_values=[1],
exp_metric_num=2)
assert result1, "check should pass when exp_metric_num matches the data"
result2 = check_query_range_result_enum(query_range_result_ok,
good_values=[1],
exp_metric_num=3)
assert not result2, "check should fail when exp_metric_num doesn't match"
def test_check_query_range_result_single_error(
query_range_result_single_error):
"""
The function finds single error in query_range_result_single_error data,
assuming 1 is a good value.
"""
result1 = check_query_range_result_enum(query_range_result_single_error,
good_values=[1])
assert not result1
result2 = check_query_range_result_enum(query_range_result_single_error,
good_values=[1, 0])
assert result2, "assuming both 1 and 0 are good values, check should pass"
def test_check_query_range_result_exp_delay(query_range_result_delay_60s):
"""
Check that exp_metric_num is taken into account, so that initial bad values
are ignored.
"""
result1 = check_query_range_result_enum(query_range_result_delay_60s,
good_values=[1],
bad_values=[0])
assert not result1, "without specifying exp_delay validation should fail"
result2 = check_query_range_result_enum(query_range_result_delay_60s,
good_values=[1],
bad_values=[0],
exp_delay=60)
assert result2, "taking exp_delay into account, validation should pass"
def test_check_query_range_result_simple_fail(query_range_result_ok):
"""
Assuming 0 is a good value, the validation should fail.
"""
result = check_query_range_result_enum(query_range_result_ok,
good_values=[0])
assert not result
def test_check_query_range_result_simple(query_range_result_ok):
"""
The function validates query_range_result_ok data assuming 1 is a good
value.
"""
assert check_query_range_result_enum(query_range_result_ok,
good_values=[1])
def test_check_query_range_result_exp_good_time(
query_range_result_bad_last_90s):
"""
Check that exp_good_time is taken into account, so that initial bad values
are ignored if appear after the good time passess.
"""
result1 = check_query_range_result_enum(query_range_result_bad_last_90s,
good_values=[1],
bad_values=[0])
assert not result1, "without exp_good_time validation should fail"
result2 = check_query_range_result_enum(
query_range_result_bad_last_90s,
good_values=[1],
bad_values=[0],
exp_good_time=150,
)
assert result2, "taking exp_good_time into account, validation should pass"
def test_check_query_range_result_null():
"""
The function does't throw any exception and returns true when executed
with empty arguments.
"""
assert check_query_range_result_enum({}, [])
def test_monitoring_shows_osd_down(measure_stop_ceph_osd):
"""
Make sure simple problems with OSD daemons are reported via OCP Prometheus.
"""
prometheus = PrometheusAPI()
# time (in seconds) for monitoring to notice the change
expected_delay = 60
affected_osd = measure_stop_ceph_osd["result"]
# translate this into ceph daemon name
ceph_daemon = "osd.{}".format(int(affected_osd[len("rook-ceph-osd-"):]))
logger.info(
f"affected osd was {affected_osd}, aka {ceph_daemon} ceph daemon")
logger.info("let's check that ceph health was affected")
health_result = prometheus.query_range(
query="ceph_health_status",
start=measure_stop_ceph_osd["start"],
end=measure_stop_ceph_osd["stop"],
step=15,
)
health_validation = check_query_range_result_enum(
result=health_result,
good_values=[1],
bad_values=[0],
exp_metric_num=1,
exp_delay=expected_delay,
)
health_msg = "health status should be affected by missing osd"
logger.info("let's check that osd up value was affected")
osd_up_result = prometheus.query_range(
query='ceph_osd_up{ceph_daemon="%s"}' % ceph_daemon,
start=measure_stop_ceph_osd["start"],
end=measure_stop_ceph_osd["stop"],
step=15,
)
osd_up_validation = check_query_range_result_enum(
result=osd_up_result,
good_values=[0],
bad_values=[1],
exp_metric_num=1,
exp_delay=expected_delay,
)
osd_up_msg = "ceph_osd_up value should be affected by missing osd"
logger.info("let's check that osd in value was not affected")
# osd in value is not affected because we just stopped the osd, we
# haven't removed it from the luster
osd_in_result = prometheus.query_range(
query='ceph_osd_in{ceph_daemon="%s"}' % ceph_daemon,
start=measure_stop_ceph_osd["start"],
end=measure_stop_ceph_osd["stop"],
step=15,
)
osd_in_validation = check_query_range_result_enum(result=osd_in_result,
good_values=[1],
bad_values=[0],
exp_metric_num=1)
osd_in_msg = "ceph_osd_in value should not be affected by missing osd"
# checking validation results when all queries are performed makes sure
# that there is evidence for all queries in the test case logs in case of
# an assert failure
assert health_validation, health_msg
assert osd_up_validation, osd_up_msg
assert osd_in_validation, osd_in_msg
def test_monitoring_shows_mon_down(measure_stop_ceph_mon):
"""
Make sure simple problems with MON daemons are reported via OCP Prometheus.
"""
prometheus = PrometheusAPI()
# time (in seconds) for monitoring to notice the change
expected_delay = 60
# query resolution step used in this test case (number of seconds)
query_step = 15
affected_mons = measure_stop_ceph_mon["result"]
# we asked to stop just a single mon ... make this assumption explicit
assert len(affected_mons) == 1
affected_mon = affected_mons[0]
# translate this into ceph daemon name
ceph_daemon = "mon.{}".format(affected_mon[len("rook-ceph-mon-"):])
logger.info(
f"affected mon was {affected_mon}, aka {ceph_daemon} ceph daemon")
logger.info("let's check that ceph health was affected")
health_result = prometheus.query_range(
query="ceph_health_status",
start=measure_stop_ceph_mon["start"],
end=measure_stop_ceph_mon["stop"],
step=query_step,
)
health_validation = check_query_range_result_enum(
result=health_result,
good_values=[1],
bad_values=[0],
exp_metric_num=1,
exp_good_time=measure_stop_ceph_mon["min_downtime"],
exp_delay=expected_delay,
)
health_msg = "health status should be affected by missing mon"
logger.info("let's check that mon quorum status value was affected")
mon_result = prometheus.query_range(
query='ceph_mon_quorum_status{ceph_daemon="%s"}' % ceph_daemon,
start=measure_stop_ceph_mon["start"],
end=measure_stop_ceph_mon["stop"],
step=query_step,
validate=False,
)
mon_validation = check_query_range_result_enum(
result=mon_result,
good_values=[0],
bad_values=[1],
exp_metric_num=1,
exp_good_time=measure_stop_ceph_mon["min_downtime"],
exp_delay=expected_delay,
)
mon_msg = "ceph_mon_quorum_status value should be affected by missing mon"
# checking validation results when both queries are performed makes sure
# that there is evidence for both mon and health queries in the test case
# logs in case of an assert failure
assert health_validation, health_msg
assert mon_validation, mon_msg
# since we don't do strict result validation in the previous query, we
# are going to check the min. expected size of the reply explicitly, taking
# into account the min. expected downtime of the affected ceph mon
assert len(mon_result) == 1, "there should be one metric for one mon"
min_mon_samples = measure_stop_ceph_mon["min_downtime"] / query_step
mon_sample_size = len(mon_result[0]["values"])
assert mon_sample_size >= min_mon_samples
def test_monitoring_reporting_ok_when_idle(workload_idle):
"""
When nothing is happening, OCP Prometheus reports OCS status as OK.
If this test case fails, the status is either reported wrong or the
cluster is in a broken state. Either way, a failure here is not good.
"""
prometheus = PrometheusAPI()
health_result = prometheus.query_range(
query="ceph_health_status",
start=workload_idle["start"],
end=workload_idle["stop"],
step=15,
)
health_validation = check_query_range_result_enum(
result=health_result, good_values=[0], bad_values=[1], exp_metric_num=1
)
health_msg = "ceph_health_status {} report 0 (health ok) as expected"
if health_validation:
health_msg = health_msg.format("does")
logger.info(health_msg)
else:
health_msg = health_msg.format("should")
logger.error(health_msg)
mon_result = prometheus.query_range(
query="ceph_mon_quorum_status",
start=workload_idle["start"],
end=workload_idle["stop"],
step=15,
)
mon_validation = check_query_range_result_enum(
result=mon_result,
good_values=[1],
bad_values=[0],
exp_metric_num=workload_idle["result"]["mon_num"],
)
mon_msg = "ceph_mon_quorum_status {} indicate no problems with quorum"
if mon_validation:
mon_msg = mon_msg.format("does")
logger.info(mon_msg)
else:
mon_msg = mon_msg.format("should")
logger.error(mon_msg)
osd_validations = []
for metric in ("ceph_osd_up", "ceph_osd_in"):
osd_result = prometheus.query_range(
query=metric,
start=workload_idle["start"],
end=workload_idle["stop"],
step=15,
)
osd_validation = check_query_range_result_enum(
result=osd_result,
good_values=[1],
bad_values=[0],
exp_metric_num=workload_idle["result"]["osd_num"],
)
osd_validations.append(osd_validation)
osd_msg = "{} metric {} indicate no problems with OSDs"
if osd_validation:
osd_msg = osd_msg.format(metric, "does")
logger.info(osd_msg)
else:
osd_msg = osd_msg.format(metric, "should")
logger.error(osd_msg)
# after logging everything properly, make the test fail if necessary
# see ERRORs reported in the test log for details
assert health_validation, health_msg
assert mon_validation, mon_msg
osds_msg = "ceph_osd_{up,in} metrics should indicate no OSD issues"
assert all(osd_validations), osds_msg