def test_ping_failure_counter_reset_after_restart(self):
"""
When a daemon stops responding and gets restarted after 5 failed pings,
it will wait for another 5 failed pings before it will be restarted
again.
"""
clock = Clock()
dog = WatchDog(clock,
broker=AsynchronousPingDaemon("test-broker"),
monitor=BoringDaemon("test-monitor"),
manager=BoringDaemon("test-manager"))
dog.start_monitoring()
for i in range(5):
clock.advance(5)
dog.broker.fire_running(False)
self.assertEqual(dog.broker.boots, ["stop", "start"])
for i in range(4):
clock.advance(5)
dog.broker.fire_running(False)
self.assertEqual(dog.broker.boots, ["stop", "start"])
clock.advance(5)
dog.broker.fire_running(False)
self.assertEqual(dog.broker.boots, ["stop", "start", "stop", "start"])
def test_counted_ping_failures_reset_on_success(self):
"""
When a failing ping is followed by a successful ping, it will then
require 5 more ping failures to restart the daemon.
"""
clock = Clock()
dog = WatchDog(clock,
broker=AsynchronousPingDaemon("test-broker"),
monitor=AsynchronousPingDaemon("test-monitor"),
manager=AsynchronousPingDaemon("test-manager"))
dog.start_monitoring()
clock.advance(5)
dog.broker.fire_running(False)
dog.monitor.fire_running(True)
dog.manager.fire_running(True)
clock.advance(5)
dog.broker.fire_running(True)
dog.monitor.fire_running(True)
dog.manager.fire_running(True)
for i in range(4):
clock.advance(5)
dog.broker.fire_running(False)
dog.monitor.fire_running(True)
dog.manager.fire_running(True)
self.assertEqual(dog.broker.boots, [])
clock.advance(5)
dog.broker.fire_running(False)
dog.monitor.fire_running(True)
dog.manager.fire_running(True)
self.assertEqual(dog.broker.boots, [STOP, START])
def test_die_when_broker_unavailable(self):
"""
If the broker is not running, the client should still be able to shut
down.
"""
self.log_helper.ignore_errors(
"Couldn't request that broker gracefully shut down; "
"killing forcefully.")
clock = Clock()
dog = WatchDog(clock,
broker=BoringDaemon("test-broker"),
monitor=BoringDaemon("test-monitor"),
manager=BoringDaemon("test-manager"))
# request_exit returns False when there's no broker, as tested by
# DaemonTest.test_request_exit_without_broker
dog.broker.request_exit = lambda: succeed(False)
# The manager's wait method never fires its deferred because nothing
# told it to die because the broker is dead!
manager_result = Deferred()
dog.manager.wait = lambda: manager_result
def stop():
manager_result.callback(True)
return succeed(True)
dog.manager.stop = stop
result = dog.request_exit()
return result
def test_wait_for_stop_before_ping(self):
"""
When a daemon times out and the watchdog restarts it, it should not be
pinged until after the restart completes.
"""
clock = Clock()
dog = WatchDog(clock,
broker=AsynchronousPingDaemon("test-broker"),
monitor=BoringDaemon("test-monitor"),
manager=BoringDaemon("test-manager"))
stop_result = Deferred()
dog.broker.stop = lambda: stop_result
dog.start_monitoring()
for i in range(5):
clock.advance(5)
dog.broker.fire_running(False)
self.assertEqual(dog.broker.boots, [])
self.assertEqual(dog.broker.pings, 5)
clock.advance(5) # wait some more to see if a ping happens
self.assertEqual(dog.broker.pings, 5)
stop_result.callback(None)
self.assertEqual(dog.broker.boots, ["start"])
clock.advance(5)
self.assertEqual(dog.broker.pings, 6)
def test_request_exit(self):
"""request_exit() asks the broker to exit.
The broker itself is responsible for notifying other plugins to exit.
When the deferred returned from request_exit fires, the process should
definitely be gone.
"""
self.setup_daemons_mocks()
self.setup_request_exit()
result = WatchDog(config=self.config).request_exit()
result.addCallback(lambda _: self.assert_request_exit())
return result
def test_exiting_during_outstanding_ping_works(self):
"""
This is a regression test. Some code called .cancel() on a timed call
without checking if it was active first. Asynchronous is_running will
cause the scheduled call to exist but already fired.
"""
clock = Clock()
dog = WatchDog(clock,
broker=BoringDaemon("test-broker"),
monitor=BoringDaemon("test-monitor"),
manager=AsynchronousPingDaemon("test-manager"))
dog.start_monitoring()
clock.advance(5)
return dog.request_exit()
def test_start_limited_daemons(self):
"""
start only starts the daemons which are actually enabled.
"""
self.setup_daemons_mocks()
clock = Clock()
dog = WatchDog(
clock, enabled_daemons=[self.broker_factory], config=self.config)
dog.start()
self.broker.start.assert_called_once_with()
self.monitor.start.assert_not_called()
self.manager.start.assert_not_called()
def test_check_running_many(self):
self.setup_daemons_mocks()
self.broker.is_running.return_value = succeed(True)
self.monitor.is_running.return_value = succeed(True)
self.manager.is_running.return_value = succeed(True)
result = WatchDog(config=self.config).check_running()
def got_result(r):
self.assertEqual([daemon.program for daemon in r],
["landscape-broker", "landscape-monitor",
"landscape-manager"])
self.assert_daemons_mocks()
return result.addCallback(got_result)
def test_check_running_limited_daemons(self):
"""
When the user has explicitly asked not to run some daemons, those
daemons which are not being run should not checked.
"""
self.setup_daemons_mocks()
self.broker.is_running.return_value = succeed(True)
result = WatchDog(enabled_daemons=[self.broker_factory],
config=self.config).check_running()
def got_result(r):
self.assertEqual(len(r), 1)
self.assertEqual(r[0].program, "landscape-broker")
return result.addCallback(got_result)
def test_start_and_stop_daemons(self):
"""The WatchDog will start all daemons, starting with the broker."""
self.setup_daemons_mocks()
self.broker.start()
self.monitor.start()
self.manager.start()
self.setup_request_exit()
clock = Clock()
dog = WatchDog(clock, config=self.config)
dog.start()
clock.advance(0)
result = dog.request_exit()
result.addCallback(lambda _: self.assert_request_exit())
return result
def test_ping_is_not_rescheduled_until_pings_complete(self):
clock = Clock()
dog = WatchDog(clock,
broker=AsynchronousPingDaemon("test-broker"),
monitor=AsynchronousPingDaemon("test-monitor"),
manager=AsynchronousPingDaemon("test-manager"))
dog.start_monitoring()
clock.advance(5)
for daemon in dog.daemons:
self.assertEqual(daemon.pings, 1)
clock.advance(5)
for daemon in dog.daemons:
self.assertEqual(daemon.pings, 1)
daemon.fire_running(True)
clock.advance(5)
for daemon in dog.daemons:
self.assertEqual(daemon.pings, 2)
def test_wb_log_notification(self):
"""
SIGUSR1 should cause logs to be reopened.
"""
mock_reactor = mock.Mock()
watchdog = WatchDog(reactor=mock_reactor, config=self.config)
os.kill(os.getpid(), signal.SIGUSR1)
mock_reactor.callFromThread.assert_called_once_with(
watchdog._notify_rotate_logs)
def test_limited_daemon_construction(self):
self.setup_daemons_mocks()
WatchDog(
enabled_daemons=[self.broker_factory, self.monitor_factory],
config=self.config)
self.broker_factory.assert_called_with(
mock.ANY, verbose=False, config=None)
self.monitor_factory.assert_called_with(
mock.ANY, verbose=False, config=None)
self.manager_factory.assert_not_called()
def test_wait_for_stop_before_start(self):
"""
When a daemon times out and the watchdog attempts to kill it, it should
not be restarted until the process has fully died.
"""
clock = Clock()
dog = WatchDog(clock,
broker=AsynchronousPingDaemon("test-broker"),
monitor=BoringDaemon("test-monitor"),
manager=BoringDaemon("test-manager"))
stop_result = Deferred()
dog.broker.stop = lambda: stop_result
dog.start_monitoring()
for i in range(5):
clock.advance(5)
dog.broker.fire_running(False)
self.assertEqual(dog.broker.boots, [])
stop_result.callback(None)
self.assertEqual(dog.broker.boots, ["start"])
def test_check_daemons(self):
"""
The daemons are checked to be running every so often. When N=5 of these
checks fail, the daemon will be restarted.
"""
clock = Clock()
dog = WatchDog(clock,
broker=AsynchronousPingDaemon("test-broker"),
monitor=AsynchronousPingDaemon("test-monitor"),
manager=AsynchronousPingDaemon("test-manager"))
dog.start_monitoring()
for i in range(4):
clock.advance(5)
dog.broker.fire_running(False)
dog.monitor.fire_running(True)
dog.manager.fire_running(True)
self.assertEqual(dog.broker.boots, [])
clock.advance(5)
dog.broker.fire_running(False)
dog.monitor.fire_running(True)
dog.manager.fire_running(True)
self.assertEqual(dog.broker.boots, [STOP, START])
def test_ping_reply_after_request_exit_should_not_restart_processes(self):
"""
When request_exit occurs between a ping request and response, a failing
ping response should not cause the process to be restarted.
"""
self.setup_daemons_mocks()
self.broker.start()
self.monitor.start()
self.manager.start()
monitor_ping_result = Deferred()
self.broker.is_running.return_value = succeed(True)
self.monitor.is_running.return_value = monitor_ping_result
self.manager.is_running.return_value = succeed(True)
self.setup_request_exit()
clock = Clock()
dog = WatchDog(clock, config=self.config)
dog.start()
clock.advance(0)
clock.advance(5)
result = dog.request_exit()
monitor_ping_result.callback(False)
def check(_):
# The monitor should never be explicitly stopped / restarted.
self.monitor.stop.assert_not_called()
# Start *is* called
self.monitor.start.call_count = 2
self.assert_request_exit()
return result.addCallback(check)
def test_daemon_construction(self):
"""The WatchDog sets up some daemons when constructed."""
self.setup_daemons_mocks()
WatchDog(config=self.config)
self.assert_daemons_mocks()