def testConsolidateServiceStatesQuasiHealthy(self):
"""Test consolidating state for a service with quasi-healthy checks."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
# Setup some test check results
hcname = TestHealthCheck.__name__
statuses = [
manager.HEALTHCHECK_STATUS(hcname, True, 'Quasi', ['RepairQuasi']),
manager.HEALTHCHECK_STATUS(hcname, True, '', [])]
cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {})
for i, status in enumerate(statuses):
name = '%s_%s' % (hcname, i)
cfm.service_check_results[TEST_SERVICE_NAME][name] = (
manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME,
status)
# Run and check the results.
cfm.ConsolidateServiceStates()
self.assertTrue(TEST_SERVICE_NAME in cfm.service_states)
_, health, healthchecks = cfm.service_states[TEST_SERVICE_NAME]
self.assertTrue(health)
self.assertEquals(1, len(healthchecks))
self.assertTrue(statuses[0] in healthchecks)
def testExecuteForce(self):
"""Test executing a health check by ignoring the check interval."""
self.StartPatcher(mock.patch('time.time'))
exec_time_offset = TestHealthCheckHasAttributes.CHECK_INTERVAL_SEC / 2
time.time.return_value = TEST_EXEC_TIME + exec_time_offset
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
cfm.service_checks = {
TEST_SERVICE_NAME: {
TestHealthCheckHasAttributes.__name__:
(TEST_MTIME, TestHealthCheckHasAttributes())
}
}
cfm.service_check_results = {
TEST_SERVICE_NAME: {
TestHealthCheckHasAttributes.__name__:
(manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME, None)
}
}
cfm.Execute(force=True)
_, exec_time, _ = cfm.service_check_results[TEST_SERVICE_NAME][
TestHealthCheckHasAttributes.__name__]
self.assertEquals(exec_time, TEST_EXEC_TIME + exec_time_offset)
def testCollectionExecutionCallbackCheckfiles(self):
"""Test the CollectionExecutionCallback on collecting checkfiles."""
self.StartPatcher(mock.patch('os.walk'))
os.walk.return_value = iter([[CHECKDIR, [TEST_SERVICE_NAME], []]])
self.StartPatcher(mock.patch('os.listdir'))
os.listdir.return_value = ['test_check.py']
self.StartPatcher(mock.patch('os.path.isfile'))
os.path.isfile.return_value = True
self.StartPatcher(mock.patch('imp.find_module'))
imp.find_module.return_value = (None, None, None)
self.StartPatcher(mock.patch('imp.load_module'))
myobj = TestHealthCheck()
manager.ImportFile = mock.Mock(return_value=[[myobj], TEST_MTIME])
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
cfm.CollectionExecutionCallback()
manager.ImportFile.assert_called_once_with(
TEST_SERVICE_NAME, './%s/test_check.py' % TEST_SERVICE_NAME)
self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks)
self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME],
{myobj.__class__.__name__: (TEST_MTIME, myobj)})
def testConsolidateServiceStatesHealthy(self):
"""Test consolidating state for a healthy service."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
# Setup some test check results
hcname = TestHealthCheck.__name__
hcname2 = '%s_2' % hcname
statuses = [
manager.HEALTHCHECK_STATUS(hcname, True, '', []),
manager.HEALTHCHECK_STATUS(hcname2, True, '', [])]
cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {})
cfm.service_check_results[TEST_SERVICE_NAME][hcname] = (
manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME, statuses[0])
cfm.service_check_results[TEST_SERVICE_NAME][hcname2] = (
manager.HCEXECUTION_IN_PROGRESS, TEST_EXEC_TIME, statuses[1])
# Run and check.
cfm.ConsolidateServiceStates()
self.assertTrue(TEST_SERVICE_NAME in cfm.service_states)
_, health, healthchecks = cfm.service_states.get(TEST_SERVICE_NAME)
self.assertTrue(health)
self.assertEquals(0, len(healthchecks))
def testGetStatusNonExistent(self):
"""Test the GetStatus RPC response when the service does not exist."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
self.assertFalse(TEST_SERVICE_NAME in cfm.service_states)
status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [])
self.assertEquals(status, cfm.GetStatus(TEST_SERVICE_NAME))
def testStartCollectionExecution(self):
"""Test the StartCollectionExecution method."""
plugins.Monitor = mock.Mock()
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
cfm.StartCollectionExecution()
self.assertTrue(plugins.Monitor.called)
def testActionInfoServiceNonExistent(self):
"""Test the ActionInfo RPC when the service does not exist."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
self.assertFalse(TEST_SERVICE_NAME in cfm.service_states)
expect = manager.ACTION_INFO('test', 'Service not recognized.', [], {})
result = cfm.ActionInfo(TEST_SERVICE_NAME, 'test', 'test')
self.assertEquals(expect, result)
def testRepairServiceNonExistent(self):
"""Test the RepairService RPC when the service does not exist."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
self.assertFalse(TEST_SERVICE_NAME in cfm.service_states)
expected = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [])
result = cfm.RepairService(TEST_SERVICE_NAME, 'DummyHealthcheck',
'DummyAction', [], {})
self.assertEquals(expected, result)
def testActionInfoServiceHealthy(self):
"""Test the ActionInfo RPC when the service is healthy."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
healthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, [])
cfm.service_states[TEST_SERVICE_NAME] = healthy_status
expect = manager.ACTION_INFO('test', 'Service is healthy.', [], {})
result = cfm.ActionInfo(TEST_SERVICE_NAME, 'test', 'test')
self.assertEquals(expect, result)
def testGetServiceList(self):
"""Test the GetServiceList RPC response."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
self.assertEquals([], cfm.GetServiceList())
status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, [])
cfm.service_states[TEST_SERVICE_NAME] = status
self.assertEquals([TEST_SERVICE_NAME], cfm.GetServiceList())
def testRepairServiceHealthy(self):
"""Test the RepairService RPC when the service is healthy."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
healthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, [])
cfm.service_states[TEST_SERVICE_NAME] = healthy_status
self.assertEquals(healthy_status, cfm.RepairService(TEST_SERVICE_NAME,
'HealthcheckName',
'RepairFuncName',
[], {}))
def testUpdateNonExistingHealthCheck(self):
"""Test adding a new health check to the manager."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
cfm.service_checks = {}
myobj = TestHealthCheck()
cfm.Update(TEST_SERVICE_NAME, [myobj], TEST_MTIME)
self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks)
self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME],
{myobj.__class__.__name__: (TEST_MTIME, myobj)})
def testCollectionExecutionCallbackNoChecks(self):
"""Test the CollectionExecutionCallback with no valid check files."""
self.StartPatcher(mock.patch('os.walk'))
os.walk.return_value = iter([['/checkdir/', [], ['test.py']]])
manager.ImportFile = mock.Mock(return_value=None)
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
cfm.CollectionExecutionCallback()
self.assertFalse(manager.ImportFile.called)
self.assertFalse(TEST_SERVICE_NAME in cfm.service_checks)
def testGetStatusSingleService(self):
"""Test the GetStatus RPC response for a single service."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
s1name = TEST_SERVICE_NAME
s2name = '%s_2' % s1name
status1 = manager.SERVICE_STATUS(s1name, True, [])
status2 = manager.SERVICE_STATUS(s2name, True, [])
cfm.service_states[s1name] = status1
cfm.service_states[s2name] = status2
self.assertEquals(status1, cfm.GetStatus(s1name))
self.assertEquals(status2, cfm.GetStatus(s2name))
def testGetStatusAllServices(self):
"""Test the GetStatus RPC response when no service is specified."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
s1name = TEST_SERVICE_NAME
s2name = '%s_2' % s1name
status1 = manager.SERVICE_STATUS(s1name, True, [])
status2 = manager.SERVICE_STATUS(s2name, True, [])
cfm.service_states[s1name] = status1
cfm.service_states[s2name] = status2
result = cfm.GetStatus('')
self.assertEquals(2, len(result))
self.assertTrue(all([x in result for x in [status1, status2]]))
def testUpdateExistingHealthCheck(self):
"""Test update when a health check exists and is not stale."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
myobj = TestHealthCheck()
cfm.service_checks[TEST_SERVICE_NAME] = {myobj.__class__.__name__:
(TEST_MTIME, myobj)}
myobj2 = TestHealthCheck()
cfm.Update(TEST_SERVICE_NAME, [myobj2], TEST_MTIME)
self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks)
self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME],
{myobj.__class__.__name__: (TEST_MTIME, myobj)})
def testActionInfo(self):
"""Test the ActionInfo RPC to collect information on a repair action."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
hcobj = TestHealthCheckMultipleActions()
hcname = hcobj.__class__.__name__
actions = [
hcobj.NoParams, hcobj.PositionalParams, hcobj.DefaultParams,
hcobj.MixedParams
]
cfm.service_checks[TEST_SERVICE_NAME] = {hcname: (TEST_MTIME, hcobj)}
unhealthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [
manager.HEALTHCHECK_STATUS(hcname, False, 'Always fails', actions)
])
cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status
# Test ActionInfo when the action has no parameters.
expect = manager.ACTION_INFO('NoParams', 'NoParams Action.', [], {})
self.assertEquals(
expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'NoParams'))
# Test ActionInfo when the action has only positional parameters.
expect = manager.ACTION_INFO('PositionalParams',
'PositionalParams Action.',
['x', 'y', 'z'], {})
self.assertEquals(
expect,
cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'PositionalParams'))
# Test ActionInfo when the action has only default parameters.
expect = manager.ACTION_INFO('DefaultParams', 'DefaultParams Action.',
[], {
'x': 1,
'y': 2,
'z': 3
})
self.assertEquals(
expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'DefaultParams'))
# Test ActionInfo when the action has positional and default parameters.
expect = manager.ACTION_INFO('MixedParams', 'MixedParams Action.',
['x', 'y'], {'z': 1})
self.assertEquals(
expect, cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'MixedParams'))
def testActionInfoActionNonExistent(self):
"""Test the ActionInfo RPC when the action does not exist."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
hcobj = TestHealthCheckUnhealthy()
cfm.service_checks[TEST_SERVICE_NAME] = {
hcobj.__class__.__name__: (TEST_MTIME, hcobj)}
unhealthy_status = manager.SERVICE_STATUS(
TEST_SERVICE_NAME, False,
[manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__,
False, 'Always fails', [hcobj.Repair])])
cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status
expect = manager.ACTION_INFO('test', 'Action not recognized.', [], {})
result = cfm.ActionInfo(TEST_SERVICE_NAME, hcobj.__class__.__name__,
'test')
self.assertEquals(expect, result)
def main(argv):
options = ParseArguments(argv)
options.Freeze()
# Configure logger.
SetupLogging(options.logdir)
# Configure global cherrypy parameters.
cherrypy.config.update({
'server.socket_host':
'0.0.0.0',
'server.socket_port':
remote_access.NormalizePort(options.port)
})
mobmon_appconfig = {
'/': {
'tools.staticdir.root': options.staticdir
},
'/static': {
'tools.staticdir.on': True,
'tools.staticdir.dir': ''
},
'/static/css': {
'tools.staticdir.dir': 'css'
},
'/static/js': {
'tools.staticdir.dir': 'js'
}
}
# Setup the mobmonitor
checkfile_manager = manager.CheckFileManager(checkdir=options.checkdir)
mobmonitor = MobMonitorRoot(checkfile_manager, staticdir=options.staticdir)
# Start the checkfile collection and execution background task.
checkfile_manager.StartCollectionExecution()
# Start the Mob* Monitor.
cherrypy.quickstart(mobmonitor, config=mobmon_appconfig)
def testRepairServiceInvalidActionArguments(self):
"""Test the RepairService RPC when the action arguments are invalid."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
hcobj = TestHealthCheckUnhealthy()
cfm.service_checks[TEST_SERVICE_NAME] = {
hcobj.__class__.__name__: (TEST_MTIME, hcobj)}
unhealthy_status = manager.SERVICE_STATUS(
TEST_SERVICE_NAME, False,
[manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__,
False, 'Always fails', [hcobj.Repair])])
cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status
status = cfm.GetStatus(TEST_SERVICE_NAME)
self.assertFalse(status.health)
self.assertEquals(1, len(status.healthchecks))
status = cfm.RepairService(TEST_SERVICE_NAME, hcobj.__class__.__name__,
'Repair', [1, 2, 3], {})
self.assertFalse(status.health)
self.assertEquals(1, len(status.healthchecks))
def testExecuteNonExistent(self):
"""Test executing a health check when the result is nonexistent."""
self.StartPatcher(mock.patch('time.time'))
time.time.return_value = TEST_EXEC_TIME
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
cfm.service_checks = {TEST_SERVICE_NAME:
{TestHealthCheck.__name__:
(TEST_MTIME, TestHealthCheck())}}
cfm.Execute()
resultsdict = cfm.service_check_results.get(TEST_SERVICE_NAME)
self.assertTrue(resultsdict is not None)
exec_status, exec_time, _ = resultsdict.get(TestHealthCheck.__name__,
(None, None, None))
self.assertTrue(exec_status is not None)
self.assertTrue(exec_time is not None)
self.assertEquals(exec_status, manager.HCEXECUTION_COMPLETED)
self.assertEquals(exec_time, TEST_EXEC_TIME)
def testRepairService(self):
"""Test the RepairService RPC to repair an unhealthy service."""
cfm = manager.CheckFileManager(checkdir=CHECKDIR)
hcobj = TestHealthCheckUnhealthy()
cfm.service_checks[TEST_SERVICE_NAME] = {
hcobj.__class__.__name__: (TEST_MTIME, hcobj)
}
unhealthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [
manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__, False,
'Always fails', [hcobj.Repair])
])
cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status
status = cfm.GetStatus(TEST_SERVICE_NAME)
self.assertFalse(status.health)
self.assertEquals(1, len(status.healthchecks))
status = cfm.RepairService(TEST_SERVICE_NAME, hcobj.__class__.__name__,
hcobj.Repair.__name__, [], {})
self.assertTrue(status.health)
self.assertEquals(0, len(status.healthchecks))
