def test_health_check_stale_amphora(self, session_mock, get_stale_amp_mock,
failover_mock, db_wait_mock):
amphora_health = mock.MagicMock()
amphora_health.amphora_id = AMPHORA_ID
get_stale_amp_mock.side_effect = [amphora_health, None]
exit_event = threading.Event()
hm = healthmanager.HealthManager(exit_event)
hm.health_check()
# Test DBDeadlock and RetryRequest exceptions
session_mock.reset_mock()
get_stale_amp_mock.reset_mock()
mock_session = mock.MagicMock()
session_mock.return_value = mock_session
get_stale_amp_mock.side_effect = [
db_exc.DBDeadlock,
db_exc.RetryRequest(Exception('retry_test')),
db_exc.DBConnectionError,
TestException('test')
]
# Test that a DBDeadlock does not raise an exception
self.assertIsNone(hm.health_check())
# Test that a RetryRequest does not raise an exception
self.assertIsNone(hm.health_check())
# Test that a DBConnectionError does not raise an exception
self.assertIsNone(hm.health_check())
# ... and that it waits for DB reconnection
db_wait_mock.assert_called_once()
# Other exceptions should raise
self.assertRaises(TestException, hm.health_check)
self.assertEqual(4, mock_session.rollback.call_count)
def test_health_check_stale_amphora(self, session_mock,
sleep_mock, get_stale_amp_mock,
failover_mock):
amphora_health = mock.MagicMock()
amphora_health.amphora_id = AMPHORA_ID
get_stale_amp_mock.side_effect = [amphora_health,
None,
TestException('test')]
hm = healthmanager.HealthManager()
self.assertRaises(TestException, hm.health_check)
failover_mock.assert_called_once_with(AMPHORA_ID)
# Test DBDeadlock and RetryRequest exceptions
session_mock.reset_mock()
get_stale_amp_mock.reset_mock()
mock_session = mock.MagicMock()
session_mock.return_value = mock_session
get_stale_amp_mock.side_effect = [
db_exc.DBDeadlock,
db_exc.RetryRequest(Exception('retry_test')),
TestException('test')]
self.assertRaises(TestException, hm.health_check)
self.assertEqual(3, mock_session.rollback.call_count)
def hm_health_check():
hm = health_manager.HealthManager()
while true_func():
try:
hm.health_check()
except Exception as e:
LOG.warning('Health Manager caught the following exception and '
'is restarting: {}'.format(e))
def test_health_check_nonestale_amphora(self, session_mock, sleep_mock,
get_stale_amp_mock, failover_mock):
session_mock.side_effect = [None, TestException('test')]
get_stale_amp_mock.return_value = None
hm = healthmanager.HealthManager()
self.assertRaises(TestException, hm.health_check)
self.assertFalse(failover_mock.called)
def test_health_check_nonstale_amphora(self, session_mock,
get_stale_amp_mock, failover_mock):
get_stale_amp_mock.side_effect = [None, TestException('test')]
exit_event = threading.Event()
hm = healthmanager.HealthManager(exit_event)
hm.health_check()
session_mock.assert_called_once_with(autocommit=False)
self.assertFalse(failover_mock.called)
def test_health_check_exit(self, session_mock, get_stale_amp_mock,
failover_mock):
get_stale_amp_mock.return_value = None
exit_event = threading.Event()
hm = healthmanager.HealthManager(exit_event)
hm.health_check()
session_mock.assert_called_once_with(autocommit=False)
self.assertFalse(failover_mock.called)
def test_health_check_stale_amphora(self, session_mock, sleep_mock,
get_stale_amp_mock, failover_mock):
amphora_health = mock.MagicMock()
amphora_health.amphora_id = AMPHORA_ID
session_mock.side_effect = [None, TestException('test')]
get_stale_amp_mock.side_effect = [amphora_health, None]
hm = healthmanager.HealthManager()
self.assertRaises(TestException, hm.health_check)
failover_mock.assert_called_once_with(AMPHORA_ID)
def test_health_check_db_error(self, session_mock, get_stale_amp_mock,
failover_mockv2, failover_mock):
get_stale_amp_mock.return_value = None
mock_session = mock.MagicMock()
session_mock.return_value = mock_session
session_mock.side_effect = TestException('DB Error')
exit_event = threading.Event()
hm = healthmanager.HealthManager(exit_event)
self.assertRaises(TestException, hm.health_check)
self.assertEqual(0, mock_session.rollback.call_count)
def test_health_check_exit(self, session_mock, get_stale_amp_mock,
failover_mockv2, failover_mock):
get_stale_amp_mock.return_value = None
exit_event = threading.Event()
hm = healthmanager.HealthManager(exit_event)
hm.health_check()
session_mock.assert_called_once_with(autocommit=False)
if CONF.api_settings.default_provider_driver == 'amphorav2':
self.assertFalse(failover_mockv2.called)
else:
self.assertFalse(failover_mock.called)
def test_health_check_nonstale_amphora(self, session_mock,
get_stale_amp_mock, failover_mockv2,
failover_mock):
get_stale_amp_mock.side_effect = [None, TestException('test')]
exit_event = threading.Event()
hm = healthmanager.HealthManager(exit_event)
hm.health_check()
session_mock.assert_called_once_with(autocommit=False)
if CONF.api_settings.default_provider_driver == 'amphorav2':
self.assertFalse(failover_mockv2.called)
else:
self.assertFalse(failover_mock.called)
def hm_health_check(exit_event):
hm = health_manager.HealthManager(exit_event)
@periodics.periodic(CONF.health_manager.health_check_interval,
run_immediately=True)
def periodic_health_check():
hm.health_check()
health_check = periodics.PeriodicWorker(
[(periodic_health_check, None, None)],
schedule_strategy='aligned_last_finished')
def hm_exit(*args, **kwargs):
health_check.stop()
hm.executor.shutdown()
signal.signal(signal.SIGINT, hm_exit)
LOG.debug("Pausing before starting health check")
exit_event.wait(CONF.health_manager.heartbeat_timeout)
health_check.start()
def hm_health_check():
hm = health_manager.HealthManager()
while True:
hm.health_check()
