def test__bring_up(self, mock_policy, mock_down_action):
"""Test interface with supervisor bringing up services.
"""
# pylint: disable=protected-access
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
treadmill.supervisor.wait_service.side_effect = [
subproc.CalledProcessError(supervisor.ERR_TIMEOUT, 's6-svwait'),
None,
]
mon._bring_up(mock_pol_inst.service)
supervisor.wait_service.assert_has_calls([
mock.call(mock_pol_inst.service.directory,
supervisor.ServiceWaitAction.up,
timeout=100),
mock.call(mock_pol_inst.service.directory,
supervisor.ServiceWaitAction.really_down,
timeout=60 * 1000),
])
treadmill.supervisor.control_service.assert_called_with(
mock_pol_inst.service.directory,
treadmill.supervisor.ServiceControlAction.up)
def test_run(self, mock_policy, mock_down_action, mock_dirwatch):
"""Test monitor run loop.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_down_action_inst = mock_down_action.return_value
mon = monitor.Monitor(services_dir='/some/dir',
service_dirs=('foo', 'bar'),
policy_impl=mock_policy,
down_action=mock_down_action())
mock_down_action_inst.execute.return_value = False
mock_dirwatch_inst = mock_dirwatch.return_value
def _mock_policy_down():
mon._down_reason = {'mock': 'data'}
mock_dirwatch_inst.process_events.side_effect = \
_mock_policy_down
mon.run()
mock_dirwatch_inst.add_dir.assert_called_with('/some/dir')
treadmill.monitor.Monitor._add_service.assert_has_calls([
mock.call('foo'),
mock.call('bar'),
mock.call('/some/dir/baz'),
])
self.assertEqual(treadmill.monitor.Monitor._add_service.call_count, 3)
self.assertTrue(mock_dirwatch.called)
self.assertTrue(mock_dirwatch_inst.wait_for_events.called)
self.assertTrue(mock_dirwatch_inst.process_events.called)
mock_down_action_inst.execute.assert_called_with({'mock': 'data'})
self.assertIsNone(mon._down_reason)
def test__on_created_exit(self, mock_policy, mock_down_action):
"""Test service exit event.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
mock_pol_inst.check.return_value = \
monitor.MonitorRestartPolicyResult.FAIL
mock_pol_inst.fail_reason = {'mock': 'data'}
mon._service_policies['/some/dir/svc/data/exits'] = mock_pol_inst
mon._on_created('/some/dir/svc/data/exits/1.111,2,3')
mon._on_created('/some/dir/svc/data/exits/1.112,2,3')
self.assertTrue(mock_pol_inst.check.called)
self.assertEqual(list(mon._down_reasons), [
{
'mock': 'data'
},
{
'mock': 'data'
},
])
self.assertFalse(treadmill.monitor.Monitor._add_service.called)
def test__process_success(self, mock_bring_up, mock_policy,
mock_down_action):
"""Test monitor success event.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_service_class = collections.namedtuple('MockSvc',
['name', 'directory'])
mock_pol_inst = mock_policy.return_value
mock_pol_inst.check.return_value = (
monitor.MonitorRestartPolicyResult.RESTART)
mock_pol_inst.service.return_value = (mock_service_class(
'mock_service', 'some_dir'))
mon._process(mock_pol_inst)
self.assertTrue(mock_pol_inst.check.called)
self.assertEqual(len(mon._down_reasons), 0)
mock_bring_up.assert_called()
def run(approot, config_dir):
"""Runs monitor."""
tm_env = None
if approot:
tm_env = appenv.AppEnvironment(root=approot)
mon = monitor.Monitor(tm_env=tm_env, config_dir=config_dir)
mon.run()
def test_run(self, mock_policy, mock_down_action, mock_dirwatch):
"""Test monitor run loop.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(scan_dirs=(os.path.join(os.sep, 'some', 'dir'),
os.path.join(os.sep, 'some', 'dir2')),
service_dirs=('foo', 'bar'),
policy_impl=mock_policy,
down_action=mock_down_action())
mock_down_action_inst = mock_down_action.return_value
mock_down_action_inst.execute.side_effect = [
True,
False, # First failure is fine, second stops the monitor
]
mock_dirwatch_inst = mock_dirwatch.return_value
def _mock_policy_down():
mon._down_reasons = collections.deque([
{
'mock': 'data'
},
{
'more': 'data'
},
{
'again': 'data'
},
])
mock_dirwatch_inst.process_events.side_effect = \
_mock_policy_down
mon.run()
mock_dirwatch_inst.add_dir.assert_has_calls([
mock.call(os.path.join(os.sep, 'some', 'dir')),
mock.call(os.path.join(os.sep, 'some', 'dir2'))
],
any_order=True)
treadmill.monitor.Monitor._add_service.assert_has_calls([
mock.call('foo'),
mock.call('bar'),
mock.call(os.path.join(os.sep, 'some', 'dir2', 'baz')),
mock.call(os.path.join(os.sep, 'some', 'dir', 'baz')),
],
any_order=True)
self.assertEqual(treadmill.monitor.Monitor._add_service.call_count, 4)
self.assertTrue(mock_dirwatch.called)
self.assertTrue(mock_dirwatch_inst.wait_for_events.called)
self.assertTrue(mock_dirwatch_inst.process_events.called)
mock_down_action_inst.execute.assert_has_calls([
mock.call({'mock': 'data'}),
mock.call({'more': 'data'}),
])
# Make sure the down_reasons queue wasn't cleared (since one of the
# down actions took down the monitor.
self.assertEqual(len(mon._down_reasons), 3)
def node_services(approot, scan_dir):
"""Setup a node services monitor enforcing restart policies.
"""
tm_env = appenv.AppEnvironment(root=approot)
mon = monitor.Monitor(scan_dirs=scan_dir,
service_dirs=None,
policy_impl=monitor.MonitorRestartPolicy,
down_action=monitor.MonitorNodeDown(tm_env))
mon.run()
def containers(approot, scan_dir):
"""Setup a monitor for the running containers.
"""
tm_env = appenv.AppEnvironment(root=approot)
mon = monitor.Monitor(
scan_dirs=scan_dir,
service_dirs=None,
policy_impl=monitor.MonitorRestartPolicy,
down_action=monitor.MonitorContainerCleanup(tm_env))
mon.run()
def services(approot, container_dir, service_dir):
"""Setup a services monitor enforcing restart policies.
"""
tm_env = appenv.AppEnvironment(root=approot)
mon = monitor.Monitor(
scan_dirs=None,
service_dirs=service_dir,
policy_impl=monitor.MonitorRestartPolicy,
down_action=monitor.MonitorContainerDown(container_dir),
event_hook=monitor.PresenceMonitorEventHook(tm_env))
mon.run()
def test_configure(self):
"""Test monitor run loop.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
config_dir = os.path.join(self.root, 'config')
watch_dir1 = os.path.join(self.root, 'watch', '1')
watch_dir2 = os.path.join(self.root, 'watch', '2')
fs.mkdir_safe(config_dir)
fs.mkdir_safe(watch_dir1)
fs.mkdir_safe(watch_dir2)
with io.open(os.path.join(config_dir, 'default'), 'w') as f:
f.writelines([
'{};plugin1\n'.format(watch_dir1),
'{};plugin2;{{"key": "value"}}\n'.format(watch_dir2)
])
impl1 = mock.Mock()
# W0613(unused-argument)
def _handler1(tm_env, params): # pylint: disable=W0613
return impl1
impl2 = mock.Mock()
# W0613(unused-argument)
def _handler2(tm_env, params): # pylint: disable=W0613
return impl2
treadmill.plugin_manager.load.side_effect = [_handler1, _handler2]
mock_dirwatch = mock.Mock()
treadmill.dirwatch.DirWatcher.return_value = mock_dirwatch
mock_dirwatch.wait_for_events.side_effect = [StopIteration()]
mon = monitor.Monitor(tm_env={}, config_dir=config_dir)
self.assertRaises(StopIteration, mon.run)
treadmill.plugin_manager.load.assert_has_calls([
mock.call('treadmill.tombstones', 'plugin1'),
mock.call('treadmill.tombstones', 'plugin2'),
],
any_order=True)
mock_dirwatch.add_dir.assert_has_calls([
mock.call(watch_dir1),
mock.call(watch_dir2),
],
any_order=True)
def cleaning(approot, scan_dir):
"""Setup a monitor for the running containers.
"""
tm_env = appenv.AppEnvironment(root=approot)
def _policy_factory():
return monitor.CleanupMonitorRestartPolicy(tm_env)
mon = monitor.Monitor(scan_dirs=scan_dir,
service_dirs=None,
policy_impl=_policy_factory,
down_action=monitor.MonitorNodeDown(
tm_env, prefix='cleanup-'))
mon.run()
def test__on_created_svc(self, mock_policy, mock_down_action):
"""Test new service created event.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(scan_dirs=('/some/dir', '/some/dir2'),
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mon._on_created('/some/dir2/new')
treadmill.monitor.Monitor._add_service.assert_called_with(
'/some/dir2/new')
def test_run(self):
"""Test monitor run loop.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
config_dir = os.path.join(self.root, 'config')
watch_dir1 = os.path.join(self.root, 'watch', '1')
fs.mkdir_safe(config_dir)
fs.mkdir_safe(watch_dir1)
event_file = os.path.join(watch_dir1, 'test2,12345.123,256,9')
with io.open(os.path.join(config_dir, 'default'), 'w') as f:
f.writelines(['{};plugin1\n'.format(watch_dir1)])
impl1 = mock.Mock()
impl1.execute.return_value = False
# W0613(unused-argument)
def _handler1(tm_env, params): # pylint: disable=W0613
return impl1
treadmill.plugin_manager.load.side_effect = [_handler1]
def _side_effect():
utils.touch(event_file)
return True
mock_dirwatch = mock.Mock()
treadmill.dirwatch.DirWatcher.return_value = mock_dirwatch
mock_dirwatch.wait_for_events.side_effect = [
_side_effect(), StopIteration()
]
mon = monitor.Monitor(tm_env={}, config_dir=config_dir)
self.assertRaises(StopIteration, mon.run)
impl1.execute.assert_called_with({
'return_code': 256,
'id': 'test2',
'signal': 9,
'timestamp': 12345.123,
})
self.assertTrue(os.path.exists(event_file))
def test__process_noop(self, mock_bring_up, mock_policy, mock_down_action):
"""Test monitor noop event.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
mock_pol_inst.check.return_value = \
monitor.MonitorRestartPolicyResult.NOOP
mon._process(mock_pol_inst)
self.assertTrue(mock_pol_inst.check.called)
mock_bring_up.assert_not_called()
self.assertEqual(len(mon._down_reasons), 0)
def test__on_created_exit(self, mock_policy, mock_down_action):
"""Test service exit event.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(services_dir='/some/dir',
service_dirs=(),
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
mock_pol_inst.process.return_value = False
mock_pol_inst.fail_reason = {'mock': 'data'}
mon._service_policies['/some/dir/svc/data/exits'] = mock_pol_inst
mon._on_created('/some/dir/svc/data/exits/1.111,2,3')
self.assertTrue(mock_pol_inst.process.called)
self.assertEqual(mon._down_reason, {'mock': 'data'})
self.assertFalse(treadmill.monitor.Monitor._add_service.called)
def test__bring_up_already_up(self, mock_policy, mock_down_action):
"""Test interface with supervisor bringing up services.
Service already up.
"""
# pylint: disable=protected-access
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
mon._bring_up(mock_pol_inst.service)
supervisor.wait_service.assert_called_with(
mock_pol_inst.service.directory,
supervisor.ServiceWaitAction.up,
timeout=100)
treadmill.supervisor.control_service.assert_not_called()
def test__add_service(self, mock_policy, mock_down_action, mock_dirwatch):
"""Test addition of a service to a running monitor.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(services_dir=None,
service_dirs=(),
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
mock_reg_handle = mock_pol_inst.register.return_value
mock_svc = treadmill.supervisor.open_service.return_value
mon._dirwatcher = mock_dirwatch()
mon._add_service(self.root)
treadmill.supervisor.open_service.assert_called_with(self.root)
mock_pol_inst.register.assert_called_with(mock_svc)
self.assertEqual(mon._service_policies[mock_reg_handle], mock_pol_inst)
mon._dirwatcher.add_dir.assert_called_with(mock_reg_handle)
self.assertTrue(mock_pol_inst.process.called)
def test__bring_up_no_sup(self, mock_policy, mock_down_action):
"""Test interface with supervisor bringing up services.
Service not supervised.
"""
# pylint: disable=protected-access
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
treadmill.supervisor.wait_service.side_effect = (
subproc.CalledProcessError(supervisor.ERR_NO_SUP, 's6-svwait'))
mon._bring_up(mock_pol_inst.service)
supervisor.wait_service.assert_called_with(
mock_pol_inst.service.directory,
supervisor.ServiceWaitAction.up,
timeout=100)
treadmill.supervisor.control_service.assert_not_called()
def test__process_fail(self, mock_policy, mock_down_action, mock_bring_up):
"""Test monitor fail event.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
# Disable W0212(protected-access)
# pylint: disable=W0212
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action())
mock_pol_inst = mock_policy.return_value
mock_pol_inst.check.return_value = \
monitor.MonitorRestartPolicyResult.FAIL
type(mock_pol_inst).fail_reason = mock.PropertyMock(
return_value={'a': 'test'})
mon._process(mock_pol_inst)
self.assertTrue(mock_pol_inst.check.called)
mock_bring_up.assert_not_called()
self.assertEqual(list(mon._down_reasons), [{'a': 'test'}])
def test_event_hook_success(self, mock_bring_up, mock_policy,
mock_down_action, mock_event_hook):
"""Test monitor up/down event hook.
"""
# Disable W0212(protected-access)
# pylint: disable=W0212
mock_event_hook_inst = mock_event_hook.return_value
mon = monitor.Monitor(scan_dirs='/some/dir',
service_dirs=None,
policy_impl=mock_policy,
down_action=mock_down_action(),
event_hook=mock_event_hook_inst)
mock_pol_inst = mock_policy.return_value
mock_pol_inst.check.return_value = \
monitor.MonitorRestartPolicyResult.RESTART
mon._process(mock_pol_inst)
self.assertTrue(mock_pol_inst.check.called)
self.assertEqual(len(mon._down_reasons), 0)
self.assertTrue(mock_event_hook_inst.down.called)
self.assertTrue(mock_event_hook_inst.up.called)
mock_bring_up.assert_called()
