def test__check_policy_fail_edge(self):
"""Test policy evaluation when the service failed too many times.
(edge case)
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_service_class = collections.namedtuple('MockSvc', ['name'])
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service_exits_log = self.root
mock_policy._service = mock_service_class('mock_service')
mock_policy._policy_interval = 30
mock_policy._policy_limit = 0
failure_ts = [1.111]
for ts in failure_ts:
exit_file = '%014.3f,001,000' % ts
with io.open(os.path.join(self.root, exit_file), 'a'):
pass
res = mock_policy.check()
self.assertEqual(res, monitor.MonitorRestartPolicyResult.FAIL)
self.assertEqual(mock_policy._last_timestamp, 1.111)
self.assertEqual(mock_policy._last_rc, 1)
self.assertEqual(mock_policy._last_signal, 0)
self.assertEqual(os.unlink.call_count, 0)
def test__check_policy_restart_egde(self):
"""Test policy evaluation when the service should be restarted.
(edge case)
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_service_class = collections.namedtuple('MockSvc', ['name'])
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service_exits_log = self.root
mock_policy._service = mock_service_class('mock_service')
mock_policy._policy_interval = 60
mock_policy._policy_limit = 1
failure_ts = [
1492807053.785,
1492807113.934,
]
for ts in failure_ts:
exit_file = '%014.3f,001,000' % ts
with io.open(os.path.join(self.root, exit_file), 'a'):
pass
res = mock_policy.check()
self.assertEqual(res, monitor.MonitorRestartPolicyResult.RESTART)
self.assertEqual(mock_policy._last_timestamp, 1492807113.934)
self.assertEqual(mock_policy._last_rc, 1)
self.assertEqual(mock_policy._last_signal, 0)
self.assertEqual(os.unlink.call_count, 0)
def test__check_policy_restart(self):
"""Test policy evaluation when the service should be restarted.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_service_class = collections.namedtuple('MockSvc', ['name'])
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service_exits_log = self.root
mock_policy._service = mock_service_class('mock_service')
mock_policy._policy_interval = 30
mock_policy._policy_limit = 3
failure_ts = [1, 2, 3, 75.431, 100.403, 115.871, 130.35]
for ts in failure_ts:
exit_file = '%014.3f,001,000' % ts
with io.open(os.path.join(self.root, exit_file), 'a'):
pass
res = mock_policy.check()
self.assertEqual(res, monitor.MonitorRestartPolicyResult.RESTART)
self.assertEqual(mock_policy._last_timestamp, 130.35)
self.assertEqual(mock_policy._last_rc, 1)
self.assertEqual(mock_policy._last_signal, 0)
self.assertEqual(
os.unlink.call_count,
# We should remove extra exit records past 2 times the policy limit
len(failure_ts) - 2 * mock_policy._policy_limit)
def test__check_listdir_fail(self):
"""Test policy evaluation when the exits dir was deleted.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_service_class = collections.namedtuple('MockSvc', ['name'])
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service_exits_log = self.root
mock_policy._service = mock_service_class('mock_service')
mock_policy._policy_interval = 30
mock_policy._policy_limit = 3
res = mock_policy.check()
self.assertEqual(res, monitor.MonitorRestartPolicyResult.NOOP)
def test__check_policy_noop(self):
"""Test policy evaluation when the service never exited.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service_exits_log = self.root
res = mock_policy.check()
self.assertEqual(res, monitor.MonitorRestartPolicyResult.NOOP)
self.assertIsNone(mock_policy._last_timestamp)
self.assertIsNone(mock_policy._last_rc)
self.assertIsNone(mock_policy._last_signal)
self.assertEqual(os.unlink.call_count, 0)
def test_register_failure(self):
"""Test policy / service registration failure (not policy).
"""
mock_service_class = collections.namedtuple('MockSvc',
['name', 'data_dir'])
mock_policy = monitor.MonitorRestartPolicy()
mock_service = mock_service_class(name='mock_service',
data_dir=os.path.join(self.root))
io.open.side_effect = \
IOError(errno.ENOENT, 'No such file or directory.')
res = mock_policy.register(mock_service)
# Check policy.json was read
io.open.assert_called_with(os.path.join(self.root, 'policy.json'))
treadmill.fs.mkdir_safe.assert_not_called()
# Failed registration should return None
self.assertIsNone(res)
def test_process_fail(self):
"""Test watch event processing.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_service_class = namedtuple('MockSvc', ['name', 'directory'])
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service_exits_log = self.root
mock_policy._service = mock_service_class('mock_service', 'some_dir')
mock_policy._policy_interval = 30
mock_policy._policy_limit = 3
mock_policy._check_policy.return_value = \
monitor.MonitorRestartPolicyResult.FAIL
res = mock_policy.process()
self.assertEqual(res, False)
self.assertEqual(treadmill.subproc.check_call.call_count, 0)
def test_fail_reason(self):
"""Test failure reason extraction from the policy.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_service_class = collections.namedtuple('MockSvc', ['name'])
mock_policy = monitor.MonitorRestartPolicy()
mock_policy._service = mock_service_class('mock_service')
mock_policy._last_timestamp = 42.123
mock_policy._last_rc = 2
mock_policy._last_signal = 9
self.assertEqual(
mock_policy.fail_reason, {
'service': 'mock_service',
'return_code': 2,
'signal': 9,
'timestamp': 42.123,
})
def test_register(self):
"""Test policy / service registration.
"""
mock_service_class = collections.namedtuple('MockSvc',
['name', 'data_dir'])
mock_policy = monitor.MonitorRestartPolicy()
mock_service = mock_service_class(name='mock_service',
data_dir=os.path.join(self.root))
json.load.return_value = {
'limit': 3,
'interval': 15,
}
res = mock_policy.register(mock_service)
# Check policy.json was read
io.open.assert_called_with(os.path.join(self.root, 'policy.json'))
treadmill.fs.mkdir_safe.assert_called_with(
os.path.join(self.root, 'exits'))
# Registration should return the exits folder to watch
self.assertEqual(res, os.path.join(self.root, 'exits'))