def test_all_health_passes_means_overall_health_passes(self):
"""
When all health checks pass the overall check is healthy
"""
checker = HealthChecker(
self.clock,
dict([("check{0}".format(i), mock.Mock(return_value=(True, {})))
for i in range(3)]))
d = checker.health_check()
self.assertEqual(
self.successResultOf(d), {
'healthy': True,
'check0': {
'healthy': True,
'details': {}
},
'check1': {
'healthy': True,
'details': {}
},
'check2': {
'healthy': True,
'details': {}
}
})
def test_no_checks(self):
"""
If there are no checks, HealthChecker returns healthy
"""
checker = HealthChecker(self.clock)
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {'healthy': True})
def test_check_is_timed_out(self):
"""
Each health check is timed out after 15 seconds
"""
checker = HealthChecker(
self.clock, {
'a': mock.Mock(return_value=defer.Deferred()),
'b': mock.Mock(return_value=defer.succeed((True, {})))
})
d = checker.health_check()
self.assertNoResult(d)
self.clock.advance(16)
r = self.successResultOf(d)
self.assertEqual(
r, {
'healthy': False,
'a': {
'healthy': False,
'details': {
'reason': mock.ANY
}
},
'b': {
'healthy': True,
'details': {}
}
})
self.assertIn('a health check timed out', r['a']['details']['reason'])
def test_no_checks(self):
"""
If there are no checks, HealthChecker returns healthy
"""
checker = HealthChecker(self.clock)
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {'healthy': True})
def test_invalid_check(self):
"""
If an invalid check is added, its health is unhealthy
"""
checker = HealthChecker(self.clock, {'invalid': None})
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {
'healthy': False,
'invalid': {
'healthy': False,
'details': {'reason': mock.ANY}
}
})
def test_synchronous_health_check(self):
"""
Synchronous health checks are supported
"""
checker = HealthChecker(self.clock, {'sync': mock.Mock(return_value=(True, {}))})
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {
'healthy': True,
'sync': {
'healthy': True,
'details': {}
}
})
def test_check_failure(self):
"""
If a check raises an exception, its health is unhealthy
"""
checker = HealthChecker(self.clock, {'fail': mock.Mock(side_effect=Exception)})
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {
'healthy': False,
'fail': {
'healthy': False,
'details': {'reason': matches(Contains('Exception'))}
}
})
def test_synchronous_health_check(self):
"""
Synchronous health checks are supported
"""
checker = HealthChecker(self.clock,
{'sync': mock.Mock(return_value=(True, {}))})
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {
'healthy': True,
'sync': {
'healthy': True,
'details': {}
}
})
def test_invalid_check(self):
"""
If an invalid check is added, its health is unhealthy
"""
checker = HealthChecker(self.clock, {'invalid': None})
d = checker.health_check()
self.assertEqual(
self.successResultOf(d), {
'healthy': False,
'invalid': {
'healthy': False,
'details': {
'reason': mock.ANY
}
}
})
def _test_setup(self, config, interval):
"""
SelfHeal function wrapped with locking and logging is setup to call
again using TimerService. It is setup on given interval based on
config
"""
clock = Clock()
log = mock_log()
health_checker = HealthChecker("clock", {})
from otter.tap.api import zk
from otter.util.config import config_value
selfheal = SelfHeal(clock, base_dispatcher, config_value, interval,
log)
self.patch(
zk, "locked_logged_func",
exp_func(self, ("func", "lock"), base_dispatcher, "/selfheallock",
log, "selfheal-lock-acquired", selfheal.setup))
self.patch(zk, "create_health_check", exp_func(self, "hc_func",
"lock"))
svc = setup_selfheal_service(clock, config, base_dispatcher,
health_checker, log)
self.assertIsInstance(svc, TimerService)
self.assertEqual(svc.call, ("func", (), {}))
self.assertIs(svc.clock, clock)
self.assertIs(health_checker.checks["selfheal"], "hc_func")
def test_check_failure(self):
"""
If a check raises an exception, its health is unhealthy
"""
checker = HealthChecker(self.clock,
{'fail': mock.Mock(side_effect=Exception)})
d = checker.health_check()
self.assertEqual(
self.successResultOf(d), {
'healthy': False,
'fail': {
'healthy': False,
'details': {
'reason': matches(Contains('Exception'))
}
}
})
def test_check_is_timed_out(self):
"""
Each health check is timed out after 15 seconds
"""
checker = HealthChecker(
self.clock, {'a': mock.Mock(return_value=defer.Deferred()),
'b': mock.Mock(return_value=defer.succeed((True, {})))})
d = checker.health_check()
self.assertNoResult(d)
self.clock.advance(16)
r = self.successResultOf(d)
self.assertEqual(r, {
'healthy': False,
'a': {'healthy': False, 'details': {'reason': mock.ANY}},
'b': {'healthy': True, 'details': {}}
})
self.assertIn('a health check timed out', r['a']['details']['reason'])
def test_one_failed_health_fails_overall_health(self):
"""
All health checks must pass in order for the main check to be healthy
"""
checker = HealthChecker(self.clock, {
'healthy_thing': mock.Mock(return_value=(True, {})),
'unhealthy_thing': mock.Mock(return_value=(False, {}))
})
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {
'healthy': False,
'healthy_thing': {
'healthy': True,
'details': {}
},
'unhealthy_thing': {
'healthy': False,
'details': {}
}
})
def test_one_failed_health_fails_overall_health(self):
"""
All health checks must pass in order for the main check to be healthy
"""
checker = HealthChecker(
self.clock, {
'healthy_thing': mock.Mock(return_value=(True, {})),
'unhealthy_thing': mock.Mock(return_value=(False, {}))
})
d = checker.health_check()
self.assertEqual(
self.successResultOf(d), {
'healthy': False,
'healthy_thing': {
'healthy': True,
'details': {}
},
'unhealthy_thing': {
'healthy': False,
'details': {}
}
})
def test_all_health_passes_means_overall_health_passes(self):
"""
When all health checks pass the overall check is healthy
"""
checker = HealthChecker(self.clock, dict([
("check{0}".format(i), mock.Mock(return_value=(True, {})))
for i in range(3)
]))
d = checker.health_check()
self.assertEqual(self.successResultOf(d), {
'healthy': True,
'check0': {
'healthy': True,
'details': {}
},
'check1': {
'healthy': True,
'details': {}
},
'check2': {
'healthy': True,
'details': {}
}
})
def make_health_checker(*args, **kwargs):
self.health_checker = HealthChecker(*args, **kwargs)
return self.health_checker