def test_sampler_base():
class TestSampler(SamplerBase):
def __init__(self, period, clock):
self.count = 0
SamplerBase.__init__(self, period, clock)
def iterate(self):
self.count += 1
test_clock = ThreadedClock()
sampler = TestSampler(Amount(1, Time.SECONDS), clock=test_clock)
sampler.start()
assert test_clock.converge(threads=[sampler])
test_clock.assert_waiting(sampler, 1)
test_clock.tick(0.5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 0
test_clock.tick(0.5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 1
test_clock.tick(5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 6
assert not sampler.is_stopped()
sampler.stop()
# make sure that stopping the sampler short circuits any sampling
test_clock.tick(5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 6
def test_sampler_base():
class TestSampler(SamplerBase):
def __init__(self, period, clock):
self.count = 0
SamplerBase.__init__(self, period, clock)
def iterate(self):
self.count += 1
test_clock = ThreadedClock()
sampler = TestSampler(Amount(1, Time.SECONDS), clock=test_clock)
sampler.start()
assert test_clock.converge(threads=[sampler])
test_clock.assert_waiting(sampler, 1)
test_clock.tick(0.5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 0
test_clock.tick(0.5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 1
test_clock.tick(5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 6
assert not sampler.is_stopped()
sampler.stop()
# make sure that stopping the sampler short circuits any sampling
test_clock.tick(5)
assert test_clock.converge(threads=[sampler])
assert sampler.count == 6
def test_sleep_0():
clock = ThreadedClock(0)
event = threading.Event()
def run():
clock.sleep(0)
event.set()
th = threading.Thread(target=run)
th.daemon = True
th.start()
assert clock.converge(threads=[th])
assert event.is_set()
def test_sleep_0():
clock = ThreadedClock(0)
event = threading.Event()
def run():
clock.sleep(0)
event.set()
th = threading.Thread(target=run)
th.daemon = True
th.start()
assert clock.converge(threads=[th])
assert event.is_set()
def test_with_events(num_threads):
event = threading.Event()
hits = []
hits_before, hits_after = 0, 0
clock = ThreadedClock(0)
def hit_me():
clock.sleep(0.1)
hits.append(True)
threads = []
for _ in range(num_threads):
th = threading.Thread(target=hit_me)
th.daemon = True
th.start()
threads.append(th)
clock.converge(threads=threads)
for th in threads:
clock.assert_waiting(th, 0.1)
clock.tick(0.05)
clock.converge(threads=threads)
hits_before += len(hits)
with pytest.raises(AssertionError):
clock.assert_waiting(threads[0], 234)
clock.tick(0.05)
clock.converge(threads=threads)
hits_after += len(hits)
for th in threads:
clock.assert_not_waiting(th)
with pytest.raises(AssertionError):
clock.assert_waiting(th, 0.1)
assert hits_before == 0
assert hits_after == num_threads
def test_with_events(num_threads):
event = threading.Event()
hits = []
hits_before, hits_after = 0, 0
clock = ThreadedClock(0)
def hit_me():
clock.sleep(0.1)
hits.append(True)
threads = []
for _ in range(num_threads):
th = threading.Thread(target=hit_me)
th.daemon = True
th.start()
threads.append(th)
clock.converge(threads=threads)
for th in threads:
clock.assert_waiting(th, 0.1)
clock.tick(0.05)
clock.converge(threads=threads)
hits_before += len(hits)
with pytest.raises(AssertionError):
clock.assert_waiting(threads[0], 234)
clock.tick(0.05)
clock.converge(threads=threads)
hits_after += len(hits)
for th in threads:
clock.assert_not_waiting(th)
with pytest.raises(AssertionError):
clock.assert_waiting(th, 0.1)
assert hits_before == 0
assert hits_after == num_threads
def test_not_converged():
clock1 = ThreadedClock(0)
clock2 = ThreadedClock(0)
def run():
clock1.sleep(1)
clock2.sleep(1)
th = threading.Thread(target=run)
th.daemon = True
th.start()
assert clock1.converge(threads=[th])
clock1.assert_waiting(th, 1)
assert clock2.converge(threads=[th], timeout=0.1) is False
clock2.assert_not_waiting(th)
clock1.tick(1)
clock2.tick(2)
clock1.converge(threads=[th])
clock2.converge(threads=[th])
clock1.assert_not_waiting(th)
clock2.assert_not_waiting(th)
def test_not_converged():
clock1 = ThreadedClock(0)
clock2 = ThreadedClock(0)
def run():
clock1.sleep(1)
clock2.sleep(1)
th = threading.Thread(target=run)
th.daemon = True
th.start()
assert clock1.converge(threads=[th])
clock1.assert_waiting(th, 1)
assert clock2.converge(threads=[th], timeout=0.1) is False
clock2.assert_not_waiting(th)
clock1.tick(1)
clock2.tick(2)
clock1.converge(threads=[th])
clock2.converge(threads=[th])
clock1.assert_not_waiting(th)
clock2.assert_not_waiting(th)
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health, interval_secs=5, clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, 10)
assert hct.status is None
self._clock.tick(6)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(3)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(5)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(
self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=5)
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
# this is an implementation detail -- we healthcheck in the initializer and
# healthcheck in the run loop. if we ever change the implementation, expect
# this to break.
assert self._checker.health.call_count == 2
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
# 2 consecutive health check failures followed by a successful health check.
epsilon = 0.001
self._clock.tick(initial_interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 0
# 3 consecutive health check failures.
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status.status == TaskState.Value('TASK_FAILED')
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 6
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check, interval_secs=1, initial_interval_secs=1, clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health,
interval_secs=5,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, 10)
assert hct.status is None
self._clock.tick(6)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(3)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(5)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=5)
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
# this is an implementation detail -- we healthcheck in the initializer and
# healthcheck in the run loop. if we ever change the implementation, expect
# this to break.
assert self._checker.health.call_count == 2
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
# 2 consecutive health check failures followed by a successful health check.
epsilon = 0.001
self._clock.tick(initial_interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 0
# 3 consecutive health check failures.
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status.status == TaskState.Value('TASK_FAILED')
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 6
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check,
interval_secs=1,
initial_interval_secs=1,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker,
amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker,
amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.initial_interval_secs = 15
self.interval_secs = 10
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_grace_period_2x_success(self):
'''Grace period is 2 x interval and health checks succeed.'''
self.append_health_checks(True, num_calls=2)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.threaded_health_checker.running is True
hct.stop()
assert self._checker.health.call_count == 1
def test_grace_period_2x_failure_then_success(self):
'''Grace period is 2 x interval and health checks fail then succeed.'''
self.append_health_checks(False)
self.append_health_checks(True)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.threaded_health_checker.running is True
hct.stop()
assert self._checker.health.call_count == 2
def test_grace_period_2x_failure(self):
'''
Grace period is 2 x interval and all health checks fail.
Failures are ignored when in grace period.
'''
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
assert hct.threaded_health_checker.running is False
hct.stop()
assert self._checker.health.call_count == 3
def test_success_outside_grace_period(self):
'''
Health checks fail inside grace period, but pass outside and leads to success
'''
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.threaded_health_checker.running is True
hct.stop()
assert self._checker.health.call_count == 3
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
grace_period_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
hct.stop()
assert self._checker.health.call_count == 1
def test_consecutive_failures_max_failures(self):
'''Verify that a task is unhealthy after max_consecutive_failures is exceeded'''
grace_period_secs = self.initial_interval_secs
interval_secs = self.interval_secs
self.append_health_checks(True, num_calls=2)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
grace_period_secs=grace_period_secs,
max_consecutive_failures=2,
min_consecutive_successes=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.metrics.sample()['consecutive_failures'] == 0
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.metrics.sample()['consecutive_failures'] == 0
assert hct.threaded_health_checker.running is True
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 5
def test_consecutive_failures_failfast(self):
'''Verify that health check is failed fast'''
grace_period_secs = self.initial_interval_secs
interval_secs = self.interval_secs
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
grace_period_secs=grace_period_secs,
max_consecutive_failures=2,
min_consecutive_successes=2,
clock=self._clock)
hct.start()
# 3 consecutive health check failures causes fail-fast
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
# failure is ignored inside grace_period_secs
assert hct.metrics.sample()['consecutive_failures'] == 0
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
assert hct.metrics.sample()['consecutive_failures'] == 1
hct.stop()
assert self._checker.health.call_count == 2
@pytest.mark.skipif('True', reason='Flaky test (AURORA-1182)')
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check, interval_secs=1, grace_period_secs=1, clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
