def test__bring_up(self, mock_policy, mock_down_action): """Test interface with supervisor bringing up services. """ # pylint: disable=protected-access mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value treadmill.supervisor.wait_service.side_effect = [ subproc.CalledProcessError(supervisor.ERR_TIMEOUT, 's6-svwait'), None, ] mon._bring_up(mock_pol_inst.service) supervisor.wait_service.assert_has_calls([ mock.call(mock_pol_inst.service.directory, supervisor.ServiceWaitAction.up, timeout=100), mock.call(mock_pol_inst.service.directory, supervisor.ServiceWaitAction.really_down, timeout=60 * 1000), ]) treadmill.supervisor.control_service.assert_called_with( mock_pol_inst.service.directory, treadmill.supervisor.ServiceControlAction.up)
def test_run(self, mock_policy, mock_down_action, mock_dirwatch): """Test monitor run loop. """ # Disable W0212(protected-access) # pylint: disable=W0212 mock_down_action_inst = mock_down_action.return_value mon = monitor.Monitor(services_dir='/some/dir', service_dirs=('foo', 'bar'), policy_impl=mock_policy, down_action=mock_down_action()) mock_down_action_inst.execute.return_value = False mock_dirwatch_inst = mock_dirwatch.return_value def _mock_policy_down(): mon._down_reason = {'mock': 'data'} mock_dirwatch_inst.process_events.side_effect = \ _mock_policy_down mon.run() mock_dirwatch_inst.add_dir.assert_called_with('/some/dir') treadmill.monitor.Monitor._add_service.assert_has_calls([ mock.call('foo'), mock.call('bar'), mock.call('/some/dir/baz'), ]) self.assertEqual(treadmill.monitor.Monitor._add_service.call_count, 3) self.assertTrue(mock_dirwatch.called) self.assertTrue(mock_dirwatch_inst.wait_for_events.called) self.assertTrue(mock_dirwatch_inst.process_events.called) mock_down_action_inst.execute.assert_called_with({'mock': 'data'}) self.assertIsNone(mon._down_reason)
def test__on_created_exit(self, mock_policy, mock_down_action): """Test service exit event. """ # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value mock_pol_inst.check.return_value = \ monitor.MonitorRestartPolicyResult.FAIL mock_pol_inst.fail_reason = {'mock': 'data'} mon._service_policies['/some/dir/svc/data/exits'] = mock_pol_inst mon._on_created('/some/dir/svc/data/exits/1.111,2,3') mon._on_created('/some/dir/svc/data/exits/1.112,2,3') self.assertTrue(mock_pol_inst.check.called) self.assertEqual(list(mon._down_reasons), [ { 'mock': 'data' }, { 'mock': 'data' }, ]) self.assertFalse(treadmill.monitor.Monitor._add_service.called)
def test__process_success(self, mock_bring_up, mock_policy, mock_down_action): """Test monitor success event. """ # Disable W0212(protected-access) # pylint: disable=W0212 # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_service_class = collections.namedtuple('MockSvc', ['name', 'directory']) mock_pol_inst = mock_policy.return_value mock_pol_inst.check.return_value = ( monitor.MonitorRestartPolicyResult.RESTART) mock_pol_inst.service.return_value = (mock_service_class( 'mock_service', 'some_dir')) mon._process(mock_pol_inst) self.assertTrue(mock_pol_inst.check.called) self.assertEqual(len(mon._down_reasons), 0) mock_bring_up.assert_called()
def run(approot, config_dir): """Runs monitor.""" tm_env = None if approot: tm_env = appenv.AppEnvironment(root=approot) mon = monitor.Monitor(tm_env=tm_env, config_dir=config_dir) mon.run()
def test_run(self, mock_policy, mock_down_action, mock_dirwatch): """Test monitor run loop. """ # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(scan_dirs=(os.path.join(os.sep, 'some', 'dir'), os.path.join(os.sep, 'some', 'dir2')), service_dirs=('foo', 'bar'), policy_impl=mock_policy, down_action=mock_down_action()) mock_down_action_inst = mock_down_action.return_value mock_down_action_inst.execute.side_effect = [ True, False, # First failure is fine, second stops the monitor ] mock_dirwatch_inst = mock_dirwatch.return_value def _mock_policy_down(): mon._down_reasons = collections.deque([ { 'mock': 'data' }, { 'more': 'data' }, { 'again': 'data' }, ]) mock_dirwatch_inst.process_events.side_effect = \ _mock_policy_down mon.run() mock_dirwatch_inst.add_dir.assert_has_calls([ mock.call(os.path.join(os.sep, 'some', 'dir')), mock.call(os.path.join(os.sep, 'some', 'dir2')) ], any_order=True) treadmill.monitor.Monitor._add_service.assert_has_calls([ mock.call('foo'), mock.call('bar'), mock.call(os.path.join(os.sep, 'some', 'dir2', 'baz')), mock.call(os.path.join(os.sep, 'some', 'dir', 'baz')), ], any_order=True) self.assertEqual(treadmill.monitor.Monitor._add_service.call_count, 4) self.assertTrue(mock_dirwatch.called) self.assertTrue(mock_dirwatch_inst.wait_for_events.called) self.assertTrue(mock_dirwatch_inst.process_events.called) mock_down_action_inst.execute.assert_has_calls([ mock.call({'mock': 'data'}), mock.call({'more': 'data'}), ]) # Make sure the down_reasons queue wasn't cleared (since one of the # down actions took down the monitor. self.assertEqual(len(mon._down_reasons), 3)
def node_services(approot, scan_dir): """Setup a node services monitor enforcing restart policies. """ tm_env = appenv.AppEnvironment(root=approot) mon = monitor.Monitor(scan_dirs=scan_dir, service_dirs=None, policy_impl=monitor.MonitorRestartPolicy, down_action=monitor.MonitorNodeDown(tm_env)) mon.run()
def containers(approot, scan_dir): """Setup a monitor for the running containers. """ tm_env = appenv.AppEnvironment(root=approot) mon = monitor.Monitor( scan_dirs=scan_dir, service_dirs=None, policy_impl=monitor.MonitorRestartPolicy, down_action=monitor.MonitorContainerCleanup(tm_env)) mon.run()
def services(approot, container_dir, service_dir): """Setup a services monitor enforcing restart policies. """ tm_env = appenv.AppEnvironment(root=approot) mon = monitor.Monitor( scan_dirs=None, service_dirs=service_dir, policy_impl=monitor.MonitorRestartPolicy, down_action=monitor.MonitorContainerDown(container_dir), event_hook=monitor.PresenceMonitorEventHook(tm_env)) mon.run()
def test_configure(self): """Test monitor run loop. """ # Disable W0212(protected-access) # pylint: disable=W0212 config_dir = os.path.join(self.root, 'config') watch_dir1 = os.path.join(self.root, 'watch', '1') watch_dir2 = os.path.join(self.root, 'watch', '2') fs.mkdir_safe(config_dir) fs.mkdir_safe(watch_dir1) fs.mkdir_safe(watch_dir2) with io.open(os.path.join(config_dir, 'default'), 'w') as f: f.writelines([ '{};plugin1\n'.format(watch_dir1), '{};plugin2;{{"key": "value"}}\n'.format(watch_dir2) ]) impl1 = mock.Mock() # W0613(unused-argument) def _handler1(tm_env, params): # pylint: disable=W0613 return impl1 impl2 = mock.Mock() # W0613(unused-argument) def _handler2(tm_env, params): # pylint: disable=W0613 return impl2 treadmill.plugin_manager.load.side_effect = [_handler1, _handler2] mock_dirwatch = mock.Mock() treadmill.dirwatch.DirWatcher.return_value = mock_dirwatch mock_dirwatch.wait_for_events.side_effect = [StopIteration()] mon = monitor.Monitor(tm_env={}, config_dir=config_dir) self.assertRaises(StopIteration, mon.run) treadmill.plugin_manager.load.assert_has_calls([ mock.call('treadmill.tombstones', 'plugin1'), mock.call('treadmill.tombstones', 'plugin2'), ], any_order=True) mock_dirwatch.add_dir.assert_has_calls([ mock.call(watch_dir1), mock.call(watch_dir2), ], any_order=True)
def cleaning(approot, scan_dir): """Setup a monitor for the running containers. """ tm_env = appenv.AppEnvironment(root=approot) def _policy_factory(): return monitor.CleanupMonitorRestartPolicy(tm_env) mon = monitor.Monitor(scan_dirs=scan_dir, service_dirs=None, policy_impl=_policy_factory, down_action=monitor.MonitorNodeDown( tm_env, prefix='cleanup-')) mon.run()
def test__on_created_svc(self, mock_policy, mock_down_action): """Test new service created event. """ # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(scan_dirs=('/some/dir', '/some/dir2'), service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mon._on_created('/some/dir2/new') treadmill.monitor.Monitor._add_service.assert_called_with( '/some/dir2/new')
def test_run(self): """Test monitor run loop. """ # Disable W0212(protected-access) # pylint: disable=W0212 config_dir = os.path.join(self.root, 'config') watch_dir1 = os.path.join(self.root, 'watch', '1') fs.mkdir_safe(config_dir) fs.mkdir_safe(watch_dir1) event_file = os.path.join(watch_dir1, 'test2,12345.123,256,9') with io.open(os.path.join(config_dir, 'default'), 'w') as f: f.writelines(['{};plugin1\n'.format(watch_dir1)]) impl1 = mock.Mock() impl1.execute.return_value = False # W0613(unused-argument) def _handler1(tm_env, params): # pylint: disable=W0613 return impl1 treadmill.plugin_manager.load.side_effect = [_handler1] def _side_effect(): utils.touch(event_file) return True mock_dirwatch = mock.Mock() treadmill.dirwatch.DirWatcher.return_value = mock_dirwatch mock_dirwatch.wait_for_events.side_effect = [ _side_effect(), StopIteration() ] mon = monitor.Monitor(tm_env={}, config_dir=config_dir) self.assertRaises(StopIteration, mon.run) impl1.execute.assert_called_with({ 'return_code': 256, 'id': 'test2', 'signal': 9, 'timestamp': 12345.123, }) self.assertTrue(os.path.exists(event_file))
def test__process_noop(self, mock_bring_up, mock_policy, mock_down_action): """Test monitor noop event. """ # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value mock_pol_inst.check.return_value = \ monitor.MonitorRestartPolicyResult.NOOP mon._process(mock_pol_inst) self.assertTrue(mock_pol_inst.check.called) mock_bring_up.assert_not_called() self.assertEqual(len(mon._down_reasons), 0)
def test__on_created_exit(self, mock_policy, mock_down_action): """Test service exit event. """ # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(services_dir='/some/dir', service_dirs=(), policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value mock_pol_inst.process.return_value = False mock_pol_inst.fail_reason = {'mock': 'data'} mon._service_policies['/some/dir/svc/data/exits'] = mock_pol_inst mon._on_created('/some/dir/svc/data/exits/1.111,2,3') self.assertTrue(mock_pol_inst.process.called) self.assertEqual(mon._down_reason, {'mock': 'data'}) self.assertFalse(treadmill.monitor.Monitor._add_service.called)
def test__bring_up_already_up(self, mock_policy, mock_down_action): """Test interface with supervisor bringing up services. Service already up. """ # pylint: disable=protected-access mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value mon._bring_up(mock_pol_inst.service) supervisor.wait_service.assert_called_with( mock_pol_inst.service.directory, supervisor.ServiceWaitAction.up, timeout=100) treadmill.supervisor.control_service.assert_not_called()
def test__add_service(self, mock_policy, mock_down_action, mock_dirwatch): """Test addition of a service to a running monitor. """ # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(services_dir=None, service_dirs=(), policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value mock_reg_handle = mock_pol_inst.register.return_value mock_svc = treadmill.supervisor.open_service.return_value mon._dirwatcher = mock_dirwatch() mon._add_service(self.root) treadmill.supervisor.open_service.assert_called_with(self.root) mock_pol_inst.register.assert_called_with(mock_svc) self.assertEqual(mon._service_policies[mock_reg_handle], mock_pol_inst) mon._dirwatcher.add_dir.assert_called_with(mock_reg_handle) self.assertTrue(mock_pol_inst.process.called)
def test__bring_up_no_sup(self, mock_policy, mock_down_action): """Test interface with supervisor bringing up services. Service not supervised. """ # pylint: disable=protected-access mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value treadmill.supervisor.wait_service.side_effect = ( subproc.CalledProcessError(supervisor.ERR_NO_SUP, 's6-svwait')) mon._bring_up(mock_pol_inst.service) supervisor.wait_service.assert_called_with( mock_pol_inst.service.directory, supervisor.ServiceWaitAction.up, timeout=100) treadmill.supervisor.control_service.assert_not_called()
def test__process_fail(self, mock_policy, mock_down_action, mock_bring_up): """Test monitor fail event. """ # Disable W0212(protected-access) # pylint: disable=W0212 # Disable W0212(protected-access) # pylint: disable=W0212 mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action()) mock_pol_inst = mock_policy.return_value mock_pol_inst.check.return_value = \ monitor.MonitorRestartPolicyResult.FAIL type(mock_pol_inst).fail_reason = mock.PropertyMock( return_value={'a': 'test'}) mon._process(mock_pol_inst) self.assertTrue(mock_pol_inst.check.called) mock_bring_up.assert_not_called() self.assertEqual(list(mon._down_reasons), [{'a': 'test'}])
def test_event_hook_success(self, mock_bring_up, mock_policy, mock_down_action, mock_event_hook): """Test monitor up/down event hook. """ # Disable W0212(protected-access) # pylint: disable=W0212 mock_event_hook_inst = mock_event_hook.return_value mon = monitor.Monitor(scan_dirs='/some/dir', service_dirs=None, policy_impl=mock_policy, down_action=mock_down_action(), event_hook=mock_event_hook_inst) mock_pol_inst = mock_policy.return_value mock_pol_inst.check.return_value = \ monitor.MonitorRestartPolicyResult.RESTART mon._process(mock_pol_inst) self.assertTrue(mock_pol_inst.check.called) self.assertEqual(len(mon._down_reasons), 0) self.assertTrue(mock_event_hook_inst.down.called) self.assertTrue(mock_event_hook_inst.up.called) mock_bring_up.assert_called()