def testConsolidateServiceStatesQuasiHealthy(self): """Test consolidating state for a service with quasi-healthy checks.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) # Setup some test check results hcname = TestHealthCheck.__name__ statuses = [ manager.HEALTHCHECK_STATUS(hcname, True, 'Quasi', ['RepairQuasi']), manager.HEALTHCHECK_STATUS(hcname, True, '', [])] cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {}) for i, status in enumerate(statuses): name = '%s_%s' % (hcname, i) cfm.service_check_results[TEST_SERVICE_NAME][name] = ( manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME, status) # Run and check the results. cfm.ConsolidateServiceStates() self.assertTrue(TEST_SERVICE_NAME in cfm.service_states) _, health, healthchecks = cfm.service_states[TEST_SERVICE_NAME] self.assertTrue(health) self.assertEquals(1, len(healthchecks)) self.assertTrue(statuses[0] in healthchecks)
def testExecuteForce(self): """Test executing a health check by ignoring the check interval.""" self.StartPatcher(mock.patch('time.time')) exec_time_offset = TestHealthCheckHasAttributes.CHECK_INTERVAL_SEC / 2 time.time.return_value = TEST_EXEC_TIME + exec_time_offset cfm = manager.CheckFileManager(checkdir=CHECKDIR) cfm.service_checks = { TEST_SERVICE_NAME: { TestHealthCheckHasAttributes.__name__: (TEST_MTIME, TestHealthCheckHasAttributes()) } } cfm.service_check_results = { TEST_SERVICE_NAME: { TestHealthCheckHasAttributes.__name__: (manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME, None) } } cfm.Execute(force=True) _, exec_time, _ = cfm.service_check_results[TEST_SERVICE_NAME][ TestHealthCheckHasAttributes.__name__] self.assertEquals(exec_time, TEST_EXEC_TIME + exec_time_offset)
def testCollectionExecutionCallbackCheckfiles(self): """Test the CollectionExecutionCallback on collecting checkfiles.""" self.StartPatcher(mock.patch('os.walk')) os.walk.return_value = iter([[CHECKDIR, [TEST_SERVICE_NAME], []]]) self.StartPatcher(mock.patch('os.listdir')) os.listdir.return_value = ['test_check.py'] self.StartPatcher(mock.patch('os.path.isfile')) os.path.isfile.return_value = True self.StartPatcher(mock.patch('imp.find_module')) imp.find_module.return_value = (None, None, None) self.StartPatcher(mock.patch('imp.load_module')) myobj = TestHealthCheck() manager.ImportFile = mock.Mock(return_value=[[myobj], TEST_MTIME]) cfm = manager.CheckFileManager(checkdir=CHECKDIR) cfm.CollectionExecutionCallback() manager.ImportFile.assert_called_once_with( TEST_SERVICE_NAME, './%s/test_check.py' % TEST_SERVICE_NAME) self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks) self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME], {myobj.__class__.__name__: (TEST_MTIME, myobj)})
def testConsolidateServiceStatesHealthy(self): """Test consolidating state for a healthy service.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) # Setup some test check results hcname = TestHealthCheck.__name__ hcname2 = '%s_2' % hcname statuses = [ manager.HEALTHCHECK_STATUS(hcname, True, '', []), manager.HEALTHCHECK_STATUS(hcname2, True, '', [])] cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {}) cfm.service_check_results[TEST_SERVICE_NAME][hcname] = ( manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME, statuses[0]) cfm.service_check_results[TEST_SERVICE_NAME][hcname2] = ( manager.HCEXECUTION_IN_PROGRESS, TEST_EXEC_TIME, statuses[1]) # Run and check. cfm.ConsolidateServiceStates() self.assertTrue(TEST_SERVICE_NAME in cfm.service_states) _, health, healthchecks = cfm.service_states.get(TEST_SERVICE_NAME) self.assertTrue(health) self.assertEquals(0, len(healthchecks))
def testGetStatusNonExistent(self): """Test the GetStatus RPC response when the service does not exist.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) self.assertFalse(TEST_SERVICE_NAME in cfm.service_states) status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, []) self.assertEquals(status, cfm.GetStatus(TEST_SERVICE_NAME))
def testStartCollectionExecution(self): """Test the StartCollectionExecution method.""" plugins.Monitor = mock.Mock() cfm = manager.CheckFileManager(checkdir=CHECKDIR) cfm.StartCollectionExecution() self.assertTrue(plugins.Monitor.called)
def testActionInfoServiceNonExistent(self): """Test the ActionInfo RPC when the service does not exist.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) self.assertFalse(TEST_SERVICE_NAME in cfm.service_states) expect = manager.ACTION_INFO('test', 'Service not recognized.', [], {}) result = cfm.ActionInfo(TEST_SERVICE_NAME, 'test', 'test') self.assertEquals(expect, result)
def testRepairServiceNonExistent(self): """Test the RepairService RPC when the service does not exist.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) self.assertFalse(TEST_SERVICE_NAME in cfm.service_states) expected = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, []) result = cfm.RepairService(TEST_SERVICE_NAME, 'DummyHealthcheck', 'DummyAction', [], {}) self.assertEquals(expected, result)
def testActionInfoServiceHealthy(self): """Test the ActionInfo RPC when the service is healthy.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) healthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, []) cfm.service_states[TEST_SERVICE_NAME] = healthy_status expect = manager.ACTION_INFO('test', 'Service is healthy.', [], {}) result = cfm.ActionInfo(TEST_SERVICE_NAME, 'test', 'test') self.assertEquals(expect, result)
def testGetServiceList(self): """Test the GetServiceList RPC response.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) self.assertEquals([], cfm.GetServiceList()) status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, []) cfm.service_states[TEST_SERVICE_NAME] = status self.assertEquals([TEST_SERVICE_NAME], cfm.GetServiceList())
def testRepairServiceHealthy(self): """Test the RepairService RPC when the service is healthy.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) healthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, []) cfm.service_states[TEST_SERVICE_NAME] = healthy_status self.assertEquals(healthy_status, cfm.RepairService(TEST_SERVICE_NAME, 'HealthcheckName', 'RepairFuncName', [], {}))
def testUpdateNonExistingHealthCheck(self): """Test adding a new health check to the manager.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) cfm.service_checks = {} myobj = TestHealthCheck() cfm.Update(TEST_SERVICE_NAME, [myobj], TEST_MTIME) self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks) self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME], {myobj.__class__.__name__: (TEST_MTIME, myobj)})
def testCollectionExecutionCallbackNoChecks(self): """Test the CollectionExecutionCallback with no valid check files.""" self.StartPatcher(mock.patch('os.walk')) os.walk.return_value = iter([['/checkdir/', [], ['test.py']]]) manager.ImportFile = mock.Mock(return_value=None) cfm = manager.CheckFileManager(checkdir=CHECKDIR) cfm.CollectionExecutionCallback() self.assertFalse(manager.ImportFile.called) self.assertFalse(TEST_SERVICE_NAME in cfm.service_checks)
def testGetStatusSingleService(self): """Test the GetStatus RPC response for a single service.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) s1name = TEST_SERVICE_NAME s2name = '%s_2' % s1name status1 = manager.SERVICE_STATUS(s1name, True, []) status2 = manager.SERVICE_STATUS(s2name, True, []) cfm.service_states[s1name] = status1 cfm.service_states[s2name] = status2 self.assertEquals(status1, cfm.GetStatus(s1name)) self.assertEquals(status2, cfm.GetStatus(s2name))
def testGetStatusAllServices(self): """Test the GetStatus RPC response when no service is specified.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) s1name = TEST_SERVICE_NAME s2name = '%s_2' % s1name status1 = manager.SERVICE_STATUS(s1name, True, []) status2 = manager.SERVICE_STATUS(s2name, True, []) cfm.service_states[s1name] = status1 cfm.service_states[s2name] = status2 result = cfm.GetStatus('') self.assertEquals(2, len(result)) self.assertTrue(all([x in result for x in [status1, status2]]))
def testUpdateExistingHealthCheck(self): """Test update when a health check exists and is not stale.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) myobj = TestHealthCheck() cfm.service_checks[TEST_SERVICE_NAME] = {myobj.__class__.__name__: (TEST_MTIME, myobj)} myobj2 = TestHealthCheck() cfm.Update(TEST_SERVICE_NAME, [myobj2], TEST_MTIME) self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks) self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME], {myobj.__class__.__name__: (TEST_MTIME, myobj)})
def testActionInfo(self): """Test the ActionInfo RPC to collect information on a repair action.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) hcobj = TestHealthCheckMultipleActions() hcname = hcobj.__class__.__name__ actions = [ hcobj.NoParams, hcobj.PositionalParams, hcobj.DefaultParams, hcobj.MixedParams ] cfm.service_checks[TEST_SERVICE_NAME] = {hcname: (TEST_MTIME, hcobj)} unhealthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [ manager.HEALTHCHECK_STATUS(hcname, False, 'Always fails', actions) ]) cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status # Test ActionInfo when the action has no parameters. expect = manager.ACTION_INFO('NoParams', 'NoParams Action.', [], {}) self.assertEquals( expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'NoParams')) # Test ActionInfo when the action has only positional parameters. expect = manager.ACTION_INFO('PositionalParams', 'PositionalParams Action.', ['x', 'y', 'z'], {}) self.assertEquals( expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'PositionalParams')) # Test ActionInfo when the action has only default parameters. expect = manager.ACTION_INFO('DefaultParams', 'DefaultParams Action.', [], { 'x': 1, 'y': 2, 'z': 3 }) self.assertEquals( expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'DefaultParams')) # Test ActionInfo when the action has positional and default parameters. expect = manager.ACTION_INFO('MixedParams', 'MixedParams Action.', ['x', 'y'], {'z': 1}) self.assertEquals( expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'MixedParams'))
def testActionInfoActionNonExistent(self): """Test the ActionInfo RPC when the action does not exist.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) hcobj = TestHealthCheckUnhealthy() cfm.service_checks[TEST_SERVICE_NAME] = { hcobj.__class__.__name__: (TEST_MTIME, hcobj)} unhealthy_status = manager.SERVICE_STATUS( TEST_SERVICE_NAME, False, [manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__, False, 'Always fails', [hcobj.Repair])]) cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status expect = manager.ACTION_INFO('test', 'Action not recognized.', [], {}) result = cfm.ActionInfo(TEST_SERVICE_NAME, hcobj.__class__.__name__, 'test') self.assertEquals(expect, result)
def main(argv): options = ParseArguments(argv) options.Freeze() # Configure logger. SetupLogging(options.logdir) # Configure global cherrypy parameters. cherrypy.config.update({ 'server.socket_host': '0.0.0.0', 'server.socket_port': remote_access.NormalizePort(options.port) }) mobmon_appconfig = { '/': { 'tools.staticdir.root': options.staticdir }, '/static': { 'tools.staticdir.on': True, 'tools.staticdir.dir': '' }, '/static/css': { 'tools.staticdir.dir': 'css' }, '/static/js': { 'tools.staticdir.dir': 'js' } } # Setup the mobmonitor checkfile_manager = manager.CheckFileManager(checkdir=options.checkdir) mobmonitor = MobMonitorRoot(checkfile_manager, staticdir=options.staticdir) # Start the checkfile collection and execution background task. checkfile_manager.StartCollectionExecution() # Start the Mob* Monitor. cherrypy.quickstart(mobmonitor, config=mobmon_appconfig)
def testRepairServiceInvalidActionArguments(self): """Test the RepairService RPC when the action arguments are invalid.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) hcobj = TestHealthCheckUnhealthy() cfm.service_checks[TEST_SERVICE_NAME] = { hcobj.__class__.__name__: (TEST_MTIME, hcobj)} unhealthy_status = manager.SERVICE_STATUS( TEST_SERVICE_NAME, False, [manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__, False, 'Always fails', [hcobj.Repair])]) cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status status = cfm.GetStatus(TEST_SERVICE_NAME) self.assertFalse(status.health) self.assertEquals(1, len(status.healthchecks)) status = cfm.RepairService(TEST_SERVICE_NAME, hcobj.__class__.__name__, 'Repair', [1, 2, 3], {}) self.assertFalse(status.health) self.assertEquals(1, len(status.healthchecks))
def testExecuteNonExistent(self): """Test executing a health check when the result is nonexistent.""" self.StartPatcher(mock.patch('time.time')) time.time.return_value = TEST_EXEC_TIME cfm = manager.CheckFileManager(checkdir=CHECKDIR) cfm.service_checks = {TEST_SERVICE_NAME: {TestHealthCheck.__name__: (TEST_MTIME, TestHealthCheck())}} cfm.Execute() resultsdict = cfm.service_check_results.get(TEST_SERVICE_NAME) self.assertTrue(resultsdict is not None) exec_status, exec_time, _ = resultsdict.get(TestHealthCheck.__name__, (None, None, None)) self.assertTrue(exec_status is not None) self.assertTrue(exec_time is not None) self.assertEquals(exec_status, manager.HCEXECUTION_COMPLETED) self.assertEquals(exec_time, TEST_EXEC_TIME)
def testRepairService(self): """Test the RepairService RPC to repair an unhealthy service.""" cfm = manager.CheckFileManager(checkdir=CHECKDIR) hcobj = TestHealthCheckUnhealthy() cfm.service_checks[TEST_SERVICE_NAME] = { hcobj.__class__.__name__: (TEST_MTIME, hcobj) } unhealthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [ manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__, False, 'Always fails', [hcobj.Repair]) ]) cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status status = cfm.GetStatus(TEST_SERVICE_NAME) self.assertFalse(status.health) self.assertEquals(1, len(status.healthchecks)) status = cfm.RepairService(TEST_SERVICE_NAME, hcobj.__class__.__name__, hcobj.Repair.__name__, [], {}) self.assertTrue(status.health) self.assertEquals(0, len(status.healthchecks))