def _send_child_down(site: Site, log: WatchLog) -> None:
# - Set child down, expect DOWN notification
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;DOWN")
if site.core_name() == "cmc":
# CMC: Send a new check result for the parent to make the CMC create the host notification
# for the child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;")
def test_child_down_after_parent_recovers(unreachable_enabled, site: Site,
initial_state):
with WatchLog(site) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(unreachable_enabled, site, log)
# - Set parent up, expect UP notification
_send_parent_recovery(site, log)
# - Next child check DOWN, expect no notification (till next parent check confirms UP)
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;")
if site.core_name() == "cmc":
# - Set parent UP (again), expect DOWN notification for child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;"
)
def test_child_down_and_up_while_not_reachable(unreachable_enabled, site: Site,
initial_state):
with WatchLog(site) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(unreachable_enabled, site, log)
# - Set child up, expect no notification
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP")
# - Set parent up, expect UP notification
_send_parent_recovery(site, log)
def test_child_up_after_parent_recovers(scenario, site: Site, initial_state):
with WatchLog(site, scenario.log, default_timeout=scenario.log_timeout) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(scenario, site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(scenario, site, log)
# - Set parent up, expect UP notification
_send_parent_recovery(scenario, site, log)
# - Next service check UP, expect no notification (till next parent check confirms UP)
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
# - Set parent UP, expect UP notification for child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP;check-mk-notify;"
)
def test_down_child_becomes_unreachable_then_up(scenario, site: Site, initial_state):
with WatchLog(site, scenario.log, default_timeout=scenario.log_timeout) as log:
# - Set child down, expect DOWN notification
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;DOWN")
if scenario.core == "cmc":
# CMC: Send a new check result for the parent to make the CMC create the host notification
# for the child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;"
)
# - Set parent down, expect DOWN notification for parent and UNREACHABLE notification for child
site.send_host_check_result("notify-test-parent", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-parent;DOWN;HARD;1;DOWN")
# Difference beween nagios/cmc: when sending DOWN via PROCESS_HOST_CHECK_RESULT
# the nagios core needs another child down check result to report it as unreachable.
if scenario.core == "cmc":
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# TODO: Can not test this because it drains too many entries from the log. WatchLog could deal
# with this by readding the read lines after succeeded test or similar
# else:
# log.check_not_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
elif scenario.core == "nagios":
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
site.send_host_check_result(
"notify-test-child", STATE_DOWN, "DOWN", expected_state=STATE_UNREACHABLE
)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# - Set child up, expect:
# cmc: pending UP notification, and sent up notification after next parent check
# nagios: UP notification
_send_child_recovery(scenario, site, log)
# - Set parent up, expect UP notification
_send_parent_recovery(scenario, site, log)
def test_down_child_becomes_unreachable_and_down_again(
unreachable_enabled, site: Site, initial_state
):
with WatchLog(site) as log:
# - Set child down, expect DOWN notification
_send_child_down(site, log)
# - Set parent down, expect DOWN notification for parent and UNREACHABLE notification for child
_send_parent_down(site, log)
# Difference beween nagios/cmc: when sending DOWN via PROCESS_HOST_CHECK_RESULT
# the nagios core needs another child down check result to report it as unreachable.
if site.core_name() == "cmc":
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# TODO: Can not test this because it drains too many entries from the log. WatchLog could deal
# with this by readding the read lines after succeeded test or similar
# else:
# log.check_not_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
elif site.core_name() == "nagios":
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
site.send_host_check_result(
"notify-test-child", STATE_DOWN, "DOWN", expected_state=STATE_UNREACHABLE
)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# - Set parent up, expect UP notification
_send_parent_recovery(site, log)
# - Next child check DOWN
# cmc: expect no notification (till next parent check confirms UP)
# nagios: expect notification without
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;")
if site.core_name() == "cmc":
# - Set parent UP (again), expect DOWN notification for child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;"
)
def test_unreachable_child_after_parent_is_down(scenario, site: Site, initial_state):
with WatchLog(site, scenario.log, default_timeout=scenario.log_timeout) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(scenario, site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(scenario, site, log)
def test_parent_down_child_state_changes(unreachable_enabled, site: Site,
initial_state):
with WatchLog(site) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
assert site.get_host_state("notify-test-child") == STATE_UP
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# - set child up, expect UP notification
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;")
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;")
# - set child down, expect UNREACHABLE notification
assert site.get_host_state("notify-test-child") == STATE_UP
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
def test_parent_down_child_up_on_up_result(scenario, site: Site, initial_state):
with WatchLog(site, scenario.log, default_timeout=scenario.log_timeout) as log:
# - Set child down, expect DOWN notification
_send_child_down(scenario, site, log)
# - Set parent down, expect DOWN notification
_send_parent_down(scenario, site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - Set child up, expect UP notification
_send_child_recovery(scenario, site, log)
def test_parent_down_child_up_on_up_result(unreachable_enabled, site: Site, initial_state):
with WatchLog(site) as log:
# - Set child down, expect DOWN notification
_send_child_down(site, log)
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - Set child up, expect UP notification
_send_child_recovery(site, log)
def test_simple_rbn_service_notification(test_log: WatchLog,
site: Site) -> None:
site.send_service_check_result("notify-test", "PING", 2, "FAKE CRIT")
# NOTE: "] " is necessary to get the actual log line and not the external command execution
test_log.check_logged(
"] SERVICE NOTIFICATION: check-mk-notify;notify-test;PING;CRITICAL;check-mk-notify;FAKE CRIT"
)
test_log.check_logged(
"] SERVICE NOTIFICATION: hh;notify-test;PING;CRITICAL;mail;FAKE CRIT")
test_log.check_logged(
"] SERVICE NOTIFICATION RESULT: hh;notify-test;PING;OK;mail;Spooled mail to local mail transmission agent;"
)
def test_unreachable_child_down_before_parent_down(unreachable_enabled: bool,
site: Site, initial_state):
with WatchLog(site) as log:
# - Set child down, expect DOWN notification
_send_child_down(site, log)
# - Set parent down, expect DOWN notification for parent and UNREACHABLE notification for child
_send_parent_down(site, log)
if site.core_name() == "cmc":
log.check_logged(
"HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# TODO: Can not check this at the moment
# else:
# log.check_not_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
elif site.core_name() == "nagios":
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# Difference beween nagios/cmc: when sending DOWN via PROCESS_HOST_CHECK_RESULT
# the nagios core needs another child down check result to report it as unreachable.
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged(
"HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
def test_simple_rbn_host_notification(test_log: WatchLog, site: Site) -> None:
site.send_host_check_result("notify-test",
1,
"FAKE DOWN",
expected_state=1)
# NOTE: "] " is necessary to get the actual log line and not the external command execution
test_log.check_logged(
"] HOST NOTIFICATION: check-mk-notify;notify-test;DOWN;check-mk-notify;FAKE DOWN"
)
test_log.check_logged(
"] HOST NOTIFICATION: hh;notify-test;DOWN;mail;FAKE DOWN")
test_log.check_logged(
"] HOST NOTIFICATION RESULT: hh;notify-test;OK;mail;Spooled mail to local mail transmission agent;"
)
def test_log_fixture(
request, web, site: Site, fake_sendmail
): # noqa: F811 # pylint: disable=redefined-outer-name
core, log = request.param
site.set_config("CORE", core, with_restart=True)
users = {
"hh": {
"alias": "Harry Hirsch",
"password": "******",
"email": "%s@localhost" % web.site.id,
"contactgroups": ["all"],
},
}
expected_users = set(["cmkadmin", "automation"] + list(users.keys()))
web.add_htpasswd_users(users)
all_users = web.get_all_users()
assert not expected_users - set(all_users.keys())
site.live.command("[%d] STOP_EXECUTING_HOST_CHECKS" % time.time())
site.live.command("[%d] STOP_EXECUTING_SVC_CHECKS" % time.time())
web.add_host(
"notify-test",
attributes={
"ipaddress": "127.0.0.1",
},
)
web.activate_changes()
with WatchLog(site, log, default_timeout=20) as l:
yield l
site.live.command("[%d] START_EXECUTING_HOST_CHECKS" % time.time())
site.live.command("[%d] START_EXECUTING_SVC_CHECKS" % time.time())
web.delete_host("notify-test")
web.delete_htpasswd_users(list(users.keys()))
web.activate_changes()
def test_log_fixture(site: Site, fake_sendmail) -> Iterator[WatchLog]:
users = {
"hh": {
"fullname": "Harry Hirsch",
"password": "******",
"email": f"{site.id}@localhost",
"contactgroups": ["all"],
},
}
initial_users = site.openapi.get_all_users()
assert len(initial_users) == 2 # expect cmkadmin and automation user
for name, user_dict in users.items():
site.openapi.create_user(username=name, **user_dict) # type: ignore
all_users = site.openapi.get_all_users()
assert len(all_users) == len(initial_users) + len(users)
site.live.command("[%d] STOP_EXECUTING_HOST_CHECKS" % time.time())
site.live.command("[%d] STOP_EXECUTING_SVC_CHECKS" % time.time())
site.openapi.create_host(
"notify-test",
attributes={
"ipaddress": "127.0.0.1",
},
)
site.activate_changes_and_wait_for_core_reload()
with WatchLog(site, default_timeout=20) as l:
yield l
site.live.command("[%d] START_EXECUTING_HOST_CHECKS" % time.time())
site.live.command("[%d] START_EXECUTING_SVC_CHECKS" % time.time())
site.openapi.delete_host("notify-test")
for username in users:
site.openapi.delete_user(username)
site.activate_changes_and_wait_for_core_reload()
def _send_child_down_expect_unreachable(unreachable_enabled: bool, site: Site,
log: WatchLog) -> None:
assert site.get_host_state("notify-test-child") == STATE_UP
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
def _send_child_recovery(site: Site, log: WatchLog) -> None:
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
log.check_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UP")
def _send_parent_down(site: Site, log: WatchLog) -> None:
site.send_host_check_result("notify-test-parent", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-parent;DOWN;HARD;1;DOWN")