def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
# 2 consecutive health check failures followed by a successful health check.
self._clock.tick(initial_interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
# 3 consecutive health check failures.
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
thread_yield()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 6
def test_initial_interval_whatev(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def setUp(self):
self.health = mock.Mock()
self.health.return_value = (True, 'Fake')
self.sandbox = mock.Mock(spec_set=SandboxInterface)
self.sandbox.exists.return_value = True
self.sandbox.root = '/root'
self.initial_interval_secs = 1
self.interval_secs = 5
self.max_consecutive_failures = 2
self.clock = mock.Mock(spec=time)
self.clock.time.return_value = 1.0
self.health_checker = HealthChecker(self.health, None,
self.interval_secs,
self.initial_interval_secs,
self.max_consecutive_failures,
self.clock)
self.health_checker_sandbox_exists = HealthChecker(
self.health, self.sandbox, self.interval_secs,
self.initial_interval_secs, self.max_consecutive_failures,
self.clock)
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
# 2 consecutive health check failures followed by a successful health check.
epsilon = 0.001
self._clock.tick(initial_interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 0
# 3 consecutive health check failures.
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status.status == TaskState.Value('TASK_FAILED')
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 6
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=5)
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
# this is an implementation detail -- we healthcheck in the initializer and
# healthcheck in the run loop. if we ever change the implementation, expect
# this to break.
assert self._checker.health.call_count == 2
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health, interval_secs=5, clock=self._clock)
hct.start()
thread_yield()
assert hct.status is None
self._clock.tick(6)
assert hct.status is None
self._clock.tick(3)
assert hct.status is None
self._clock.tick(5)
thread_yield()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check,
interval_secs=1,
initial_interval_secs=1,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker,
amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker,
amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health,
interval_secs=5,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, 10)
assert hct.status is None
self._clock.tick(6)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(3)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(5)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
