def test_child_down_after_parent_recovers(unreachable_enabled, site: Site,
initial_state):
with WatchLog(site) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(unreachable_enabled, site, log)
# - Set parent up, expect UP notification
_send_parent_recovery(site, log)
# - Next child check DOWN, expect no notification (till next parent check confirms UP)
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;")
if site.core_name() == "cmc":
# - Set parent UP (again), expect DOWN notification for child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;"
)
def test_child_up_after_parent_recovers(scenario, site: Site, initial_state):
with WatchLog(site, scenario.log, default_timeout=scenario.log_timeout) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(scenario, site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(scenario, site, log)
# - Set parent up, expect UP notification
_send_parent_recovery(scenario, site, log)
# - Next service check UP, expect no notification (till next parent check confirms UP)
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
# - Set parent UP, expect UP notification for child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP;check-mk-notify;"
)
def test_child_down_and_up_while_not_reachable(unreachable_enabled, site: Site,
initial_state):
with WatchLog(site) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
_send_child_down_expect_unreachable(unreachable_enabled, site, log)
# - Set child up, expect no notification
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UP")
# - Set parent up, expect UP notification
_send_parent_recovery(site, log)
def _send_child_down(scenario, site: Site, log):
# - Set child down, expect DOWN notification
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;DOWN")
if scenario.core == "cmc":
# CMC: Send a new check result for the parent to make the CMC create the host notification
# for the child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;")
def test_simple_rbn_host_notification(test_log, site: Site):
site.send_host_check_result("notify-test", 1, "FAKE DOWN", expected_state=1)
# NOTE: "] " is necessary to get the actual log line and not the external command execution
test_log.check_logged(
"] HOST NOTIFICATION: check-mk-notify;notify-test;DOWN;check-mk-notify;FAKE DOWN"
)
test_log.check_logged("] HOST NOTIFICATION: hh;notify-test;DOWN;mail;FAKE DOWN")
test_log.check_logged(
"] HOST NOTIFICATION RESULT: hh;notify-test;OK;mail;Spooled mail to local mail transmission agent;"
)
def test_down_child_becomes_unreachable_then_up(scenario, site: Site, initial_state):
with WatchLog(site, scenario.log, default_timeout=scenario.log_timeout) as log:
# - Set child down, expect DOWN notification
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;DOWN")
if scenario.core == "cmc":
# CMC: Send a new check result for the parent to make the CMC create the host notification
# for the child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;"
)
# - Set parent down, expect DOWN notification for parent and UNREACHABLE notification for child
site.send_host_check_result("notify-test-parent", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-parent;DOWN;HARD;1;DOWN")
# Difference beween nagios/cmc: when sending DOWN via PROCESS_HOST_CHECK_RESULT
# the nagios core needs another child down check result to report it as unreachable.
if scenario.core == "cmc":
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# TODO: Can not test this because it drains too many entries from the log. WatchLog could deal
# with this by readding the read lines after succeeded test or similar
# else:
# log.check_not_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
elif scenario.core == "nagios":
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
site.send_host_check_result(
"notify-test-child", STATE_DOWN, "DOWN", expected_state=STATE_UNREACHABLE
)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# - Set child up, expect:
# cmc: pending UP notification, and sent up notification after next parent check
# nagios: UP notification
_send_child_recovery(scenario, site, log)
# - Set parent up, expect UP notification
_send_parent_recovery(scenario, site, log)
def _send_child_down_expect_unreachable(scenario, site: Site, log):
assert site.get_host_state("notify-test-child") == STATE_UP
site.send_host_check_result(
"notify-test-child", STATE_DOWN, "DOWN", expected_state=STATE_UNREACHABLE
)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if scenario.unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
def test_unreachable_child_down_before_parent_down(unreachable_enabled: bool,
site: Site, initial_state):
with WatchLog(site) as log:
# - Set child down, expect DOWN notification
_send_child_down(site, log)
# - Set parent down, expect DOWN notification for parent and UNREACHABLE notification for child
_send_parent_down(site, log)
if site.core_name() == "cmc":
log.check_logged(
"HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# TODO: Can not check this at the moment
# else:
# log.check_not_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
elif site.core_name() == "nagios":
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# Difference beween nagios/cmc: when sending DOWN via PROCESS_HOST_CHECK_RESULT
# the nagios core needs another child down check result to report it as unreachable.
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged(
"HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
def initial_state_fixture(site: Site, scenario):
# Before each test: Set to initial state: Both UP
site.send_host_check_result("notify-test-child", 0, "UP")
site.send_host_check_result("notify-test-parent", 0, "UP")
# Before each test: Clear logs
if scenario.core == "cmc":
# The command is processed asynchronously -> Wait for completion
inode_before = os.stat(site.path("var/check_mk/core/history")).st_ino
site.live.command("[%d] ROTATE_LOGFILE" % time.time())
def rotated_log():
try:
return inode_before != os.stat(site.path("var/check_mk/core/history")).st_ino
except OSError as e:
if e.errno == errno.ENOENT:
return False
raise e
wait_until(rotated_log, timeout=10)
else:
site.delete_file("var/nagios/nagios.log")
def test_down_child_becomes_unreachable_and_down_again(
unreachable_enabled, site: Site, initial_state
):
with WatchLog(site) as log:
# - Set child down, expect DOWN notification
_send_child_down(site, log)
# - Set parent down, expect DOWN notification for parent and UNREACHABLE notification for child
_send_parent_down(site, log)
# Difference beween nagios/cmc: when sending DOWN via PROCESS_HOST_CHECK_RESULT
# the nagios core needs another child down check result to report it as unreachable.
if site.core_name() == "cmc":
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# TODO: Can not test this because it drains too many entries from the log. WatchLog could deal
# with this by readding the read lines after succeeded test or similar
# else:
# log.check_not_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
elif site.core_name() == "nagios":
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
site.send_host_check_result(
"notify-test-child", STATE_DOWN, "DOWN", expected_state=STATE_UNREACHABLE
)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# - Set parent up, expect UP notification
_send_parent_recovery(site, log)
# - Next child check DOWN
# cmc: expect no notification (till next parent check confirms UP)
# nagios: expect notification without
site.send_host_check_result("notify-test-child", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-child;DOWN;HARD;1;")
if site.core_name() == "cmc":
# - Set parent UP (again), expect DOWN notification for child
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;DOWN;check-mk-notify;"
)
def test_parent_down_child_state_changes(unreachable_enabled, site: Site,
initial_state):
with WatchLog(site) as log:
# - Set parent down, expect DOWN notification
_send_parent_down(site, log)
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;DOWN;check-mk-notify;"
)
# - set child down, expect UNREACHABLE notification
assert site.get_host_state("notify-test-child") == STATE_UP
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
# - set child up, expect UP notification
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;")
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;")
# - set child down, expect UNREACHABLE notification
assert site.get_host_state("notify-test-child") == STATE_UP
site.send_host_check_result("notify-test-child",
STATE_DOWN,
"DOWN",
expected_state=STATE_UNREACHABLE)
log.check_logged("HOST ALERT: notify-test-child;UNREACHABLE;HARD;1;")
if unreachable_enabled:
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
else:
log.check_not_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-child;UNREACHABLE;check-mk-notify;"
)
def _send_child_recovery(scenario, site: Site, log):
site.send_host_check_result("notify-test-child", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-child;UP;HARD;1;")
log.check_logged("HOST NOTIFICATION: check-mk-notify;notify-test-child;UP")
def _send_parent_down(scenario, site: Site, log):
site.send_host_check_result("notify-test-parent", STATE_DOWN, "DOWN")
log.check_logged("HOST ALERT: notify-test-parent;DOWN;HARD;1;DOWN")
def _send_parent_recovery(site: Site, log: WatchLog) -> None:
site.send_host_check_result("notify-test-parent", STATE_UP, "UP")
log.check_logged("HOST ALERT: notify-test-parent;UP;HARD;1;")
log.check_logged(
"HOST NOTIFICATION: check-mk-notify;notify-test-parent;UP")