def test_slow_heartbeater(self, client_class, time_class, chairman):
"""Don't report missing if the current sequence number is equal to
the sequence number of the last successful ping.
"""
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
self._clients["bar"] = bar_client
children = {"bar": ServerAddress("bar", 1234)}
# send a ping. bar should get reported resurrected.
health_checker = HealthChecker("id", children, self.conf)
time_class.return_value = 0.0
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
health_checker._send_heartbeat()
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar'],
schedulers=None,
scheduler_id='id')
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
self.assertEquals(health_checker._resurrected_children, set(["bar"]))
self.assertEquals(health_checker._missing_children, set())
# call _send_report() again after 100 seconds. bar shouldn't get
# reported missing since the heartbeater hasn't send another ping.
time_class.return_value = 100.0
chairman.reset_mock()
health_checker._send_report()
self.assertFalse(chairman.return_value.report_missing.called)
self.assertFalse(chairman.return_value.report_resurrected.called)
# ping fails. now the reporter should report bar missing.
bar_client.ping.side_effect = Exception()
chairman.return_value.report_missing.return_value = \
ReportMissingResponse(result=0)
health_checker._send_heartbeat()
health_checker._send_report()
req = ReportMissingRequest(hosts=['bar'],
schedulers=None,
scheduler_id='id')
chairman.return_value.report_missing.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_resurrected.called)
def test_chairman_failure(self, client_class, time_class, chairman):
"""Reporter should retry reporting if chairman fails."""
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
self._clients["bar"] = bar_client
children = {"bar": ServerAddress("bar", 1234)}
# report_resurrected returns a non-zero value
health_checker = HealthChecker("id", children, self.conf)
time_class.return_value = 0.0
health_checker._send_heartbeat()
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=1)
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar'],
schedulers=None,
scheduler_id='id')
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
# report_resurrected throws an exception
chairman.reset_mock()
chairman.return_value.report_resurrected.side_effect = Exception()
health_checker._send_report()
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
# report succeeds
chairman.reset_mock()
chairman.return_value.report_resurrected.side_effect = None
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
health_checker._send_report()
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
# report doesn't get called anymore.
chairman.reset_mock()
health_checker._send_report()
self.assertFalse(chairman.return_value.report_resurrected.called)
self.assertFalse(chairman.return_value.report_missing.called)
def test_slow_heartbeater(self, client_class, time_class, chairman):
"""Don't report missing if the current sequence number is equal to
the sequence number of the last successful ping.
"""
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
self._clients["bar"] = bar_client
children = {"bar": ServerAddress("bar", 1234)}
# send a ping. bar should get reported resurrected.
health_checker = HealthChecker("id", children, self.conf)
time_class.return_value = 0.0
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
health_checker._send_heartbeat()
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar'], schedulers=None,
scheduler_id='id')
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
self.assertEquals(health_checker._resurrected_children, set(["bar"]))
self.assertEquals(health_checker._missing_children, set())
# call _send_report() again after 100 seconds. bar shouldn't get
# reported missing since the heartbeater hasn't send another ping.
time_class.return_value = 100.0
chairman.reset_mock()
health_checker._send_report()
self.assertFalse(chairman.return_value.report_missing.called)
self.assertFalse(chairman.return_value.report_resurrected.called)
# ping fails. now the reporter should report bar missing.
bar_client.ping.side_effect = Exception()
chairman.return_value.report_missing.return_value = \
ReportMissingResponse(result=0)
health_checker._send_heartbeat()
health_checker._send_report()
req = ReportMissingRequest(hosts=['bar'], schedulers=None,
scheduler_id='id')
chairman.return_value.report_missing.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_resurrected.called)
def test_report(self, client_class, time_class, chairman):
"""Test that resurrected and missing hosts get reported correctly"""
client_class.side_effect = self.create_fake_client
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
bar_client = MagicMock()
baz_client = MagicMock()
self._clients["bar"] = bar_client
self._clients["baz"] = baz_client
children = {
"bar": ServerAddress("bar", 1234),
"baz": ServerAddress("baz", 1234)
}
# first ping succeeds for bar and baz. they get reported resurrected.
health_checker = HealthChecker("id", children, self.conf)
time_class.return_value = 0.0
health_checker._send_heartbeat()
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar', 'baz'],
schedulers=None,
scheduler_id='id')
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
self.assertEquals(health_checker._resurrected_children,
set(["bar", "baz"]))
self.assertEquals(health_checker._missing_children, set())
# call _send_report again. this time nothing should get reported.
chairman.reset_mock()
health_checker._send_report()
self.assertFalse(chairman.return_value.report_missing.called)
self.assertFalse(chairman.return_value.report_resurrected.called)
# bar goes missing.
bar_client.ping.side_effect = Exception()
health_checker._send_heartbeat()
time_class.return_value = 100.0
chairman.return_value.report_missing.return_value = \
ReportMissingResponse(result=0)
health_checker._send_report()
req = ReportMissingRequest(hosts=['bar'],
schedulers=None,
scheduler_id='id')
chairman.return_value.report_missing.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_resurrected.called)
# bar comes back
chairman.reset_mock()
bar_client.ping.side_effect = None
time_class.return_value = 200.0
health_checker._send_heartbeat()
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar'],
schedulers=None,
scheduler_id='id')
self.assertFalse(chairman.return_value.report_missing.called)
chairman.return_value.report_resurrected.assert_called_once_with(req)
def test_chairman_failure(self, client_class, time_class, chairman):
"""Reporter should retry reporting if chairman fails."""
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
self._clients["bar"] = bar_client
children = {"bar": ServerAddress("bar", 1234)}
# report_resurrected returns a non-zero value
health_checker = HealthChecker("id", children, self.conf)
time_class.return_value = 0.0
health_checker._send_heartbeat()
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=1)
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar'], schedulers=None,
scheduler_id='id')
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
# report_resurrected throws an exception
chairman.reset_mock()
chairman.return_value.report_resurrected.side_effect = Exception()
health_checker._send_report()
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
# report succeeds
chairman.reset_mock()
chairman.return_value.report_resurrected.side_effect = None
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
health_checker._send_report()
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
# report doesn't get called anymore.
chairman.reset_mock()
health_checker._send_report()
self.assertFalse(chairman.return_value.report_resurrected.called)
self.assertFalse(chairman.return_value.report_missing.called)
def test_report(self, client_class, time_class, chairman):
"""Test that resurrected and missing hosts get reported correctly"""
client_class.side_effect = self.create_fake_client
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
bar_client = MagicMock()
baz_client = MagicMock()
self._clients["bar"] = bar_client
self._clients["baz"] = baz_client
children = {"bar": ServerAddress("bar", 1234),
"baz": ServerAddress("baz", 1234)}
# first ping succeeds for bar and baz. they get reported resurrected.
health_checker = HealthChecker("id", children, self.conf)
time_class.return_value = 0.0
health_checker._send_heartbeat()
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar', 'baz'], schedulers=None,
scheduler_id='id')
chairman.return_value.report_resurrected.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_missing.called)
self.assertEquals(health_checker._resurrected_children,
set(["bar", "baz"]))
self.assertEquals(health_checker._missing_children, set())
# call _send_report again. this time nothing should get reported.
chairman.reset_mock()
health_checker._send_report()
self.assertFalse(chairman.return_value.report_missing.called)
self.assertFalse(chairman.return_value.report_resurrected.called)
# bar goes missing.
bar_client.ping.side_effect = Exception()
health_checker._send_heartbeat()
time_class.return_value = 100.0
chairman.return_value.report_missing.return_value = \
ReportMissingResponse(result=0)
health_checker._send_report()
req = ReportMissingRequest(hosts=['bar'], schedulers=None,
scheduler_id='id')
chairman.return_value.report_missing.assert_called_once_with(req)
self.assertFalse(chairman.return_value.report_resurrected.called)
# bar comes back
chairman.reset_mock()
bar_client.ping.side_effect = None
time_class.return_value = 200.0
health_checker._send_heartbeat()
chairman.return_value.report_resurrected.return_value = \
ReportResurrectedResponse(result=0)
health_checker._send_report()
req = ReportResurrectedRequest(hosts=['bar'], schedulers=None,
scheduler_id='id')
self.assertFalse(chairman.return_value.report_missing.called)
chairman.return_value.report_resurrected.assert_called_once_with(req)
def test_heartbeat(self, client_class, time_class):
"""Test that sequence number and timestamp get updated correctly
after sending heartbeat."""
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
baz_client = MagicMock()
self._clients["bar"] = bar_client
self._clients["baz"] = baz_client
children = {
"bar": ServerAddress("bar", 1234),
"baz": ServerAddress("baz", 1234)
}
# make sure things are initialized properly
time_class.return_value = 0.0
health_checker = HealthChecker("id", children, self.conf)
self.assertEquals(health_checker._seqnum, 0)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (0, 0.0))
self.assertEquals(health_checker._last_update["baz"], (0, 0.0))
# send a ping
time_class.return_value = 10.0
health_checker._send_heartbeat()
self.assertEquals(health_checker._seqnum, 1)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (1, 10.0))
self.assertEquals(health_checker._last_update["baz"], (1, 10.0))
# send another ping. ping to baz fails.
time_class.return_value = 20.0
baz_client.ping.side_effect = Exception()
health_checker._send_heartbeat()
self.assertEquals(health_checker._seqnum, 2)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (2, 20.0))
self.assertEquals(health_checker._last_update["baz"], (1, 10.0))
# send another ping. ping to bar fails.
time_class.return_value = 30.0
bar_client.ping.side_effect = Exception()
baz_client.ping.side_effect = None
health_checker._send_heartbeat()
self.assertEquals(health_checker._seqnum, 3)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (2, 20.0))
self.assertEquals(health_checker._last_update["baz"], (3, 30.0))
def test_heartbeat(self, client_class, time_class):
"""Test that sequence number and timestamp get updated correctly
after sending heartbeat."""
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
baz_client = MagicMock()
self._clients["bar"] = bar_client
self._clients["baz"] = baz_client
children = {"bar": ServerAddress("bar", 1234),
"baz": ServerAddress("baz", 1234)}
# make sure things are initialized properly
time_class.return_value = 0.0
health_checker = HealthChecker("id", children, self.conf)
self.assertEquals(health_checker._seqnum, 0)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (0, 0.0))
self.assertEquals(health_checker._last_update["baz"], (0, 0.0))
# send a ping
time_class.return_value = 10.0
health_checker._send_heartbeat()
self.assertEquals(health_checker._seqnum, 1)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (1, 10.0))
self.assertEquals(health_checker._last_update["baz"], (1, 10.0))
# send another ping. ping to baz fails.
time_class.return_value = 20.0
baz_client.ping.side_effect = Exception()
health_checker._send_heartbeat()
self.assertEquals(health_checker._seqnum, 2)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (2, 20.0))
self.assertEquals(health_checker._last_update["baz"], (1, 10.0))
# send another ping. ping to bar fails.
time_class.return_value = 30.0
bar_client.ping.side_effect = Exception()
baz_client.ping.side_effect = None
health_checker._send_heartbeat()
self.assertEquals(health_checker._seqnum, 3)
self.assertEquals(len(health_checker._last_update), 2)
self.assertEquals(health_checker._last_update["bar"], (2, 20.0))
self.assertEquals(health_checker._last_update["baz"], (3, 30.0))
