def test_announcer_under_abnormal_circumstances():
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset.join = MagicMock()
mock_serverset.join.side_effect = [
KazooException('Whoops the ensemble is down!'),
'member0001',
]
mock_serverset.cancel = MagicMock()
endpoint = Endpoint('localhost', 12345)
clock = ThreadedClock(31337.0)
announcer = Announcer(
mock_serverset, endpoint, clock=clock, exception_wait=Amount(2, Time.SECONDS))
announcer.start()
try:
clock.tick(1.0)
assert announcer.disconnected_time() == 1.0
clock.tick(2.0)
assert announcer.disconnected_time() == 0.0, (
'Announcer should recover after an exception thrown internally.')
assert announcer._membership == 'member0001'
finally:
announcer.stop()
def test_ttl_decrement_works():
clock = ThreadedClock()
pps = TestPingPongServer('foo', 31337, clock=clock)
pps.ping('hello world', ttl=1)
clock.tick(TestPingPongServer.PING_DELAY.as_(Time.SECONDS))
pps.expect_calls()
def test_caching():
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
m = mox.Mox()
m.StubOutWithMock(os.path, 'getmtime')
os.path.getmtime(mox.IgnoreArg()).MultipleTimes().AndReturn(0)
m.ReplayAll()
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert os.path.exists(web.translate_url(URL))
assert opener.opened.is_set()
opener.clear()
assert web.expired(URL, ttl=0.5) is False
clock.tick(1)
assert web.expired(URL, ttl=0.5)
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert not opener.opened.is_set()
with contextlib.closing(web.open(URL, ttl=0.5)) as fp:
assert fp.read() == DATA
assert opener.opened.is_set(), 'expect expired url to cause http get'
m.UnsetStubs()
m.VerifyAll()
def test_caching(getmtime_mock):
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
getmtime_mock.return_value = 0
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert os.path.exists(web.translate_url(URL))
assert opener.opened.is_set()
opener.clear()
assert web.expired(URL, ttl=0.5) is False
clock.tick(1)
assert web.expired(URL, ttl=0.5)
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert not opener.opened.is_set()
with contextlib.closing(web.open(URL, ttl=0.5)) as fp:
assert fp.read() == DATA
assert opener.opened.is_set(), 'expect expired url to cause http get'
def test_ping():
clock = ThreadedClock()
pps = TestPingPongServer('foo', 31337, clock=clock)
pps.ping('hello world', ttl=2)
clock.tick(TestPingPongServer.PING_DELAY.as_(Time.SECONDS))
pps.expect_calls(('ping', 'hello world', 1))
pps.ping('hello world', ttl=3)
clock.tick(TestPingPongServer.PING_DELAY.as_(Time.SECONDS))
pps.expect_calls(('ping', 'hello world', 2))
def test_defer():
clock = ThreadedClock()
DELAY = 3
results = Queue(maxsize=1)
def func():
results.put_nowait('success')
defer(func, delay=DELAY, clock=clock)
with Timer(clock=clock) as timer:
clock.tick(4)
assert results.get() == 'success'
assert timer.elapsed >= DELAY
def test_sleep_0():
clock = ThreadedClock(0)
event = threading.Event()
def run():
clock.sleep(0)
event.set()
th = threading.Thread(target=run)
th.daemon = True
th.start()
assert clock.converge(threads=[th])
assert event.is_set()
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def test_defer():
DELAY = 3
clock = ThreadedClock()
results = Queue(maxsize=1)
def func():
results.put_nowait('success')
defer(func, delay=DELAY, clock=clock)
with Timer(clock=clock) as timer:
with pytest.raises(Empty):
results.get_nowait()
clock.tick(DELAY + 1)
assert results.get() == 'success'
assert timer.elapsed == DELAY + 1
def test_announcer_under_abnormal_circumstances():
mock_serverset = mock.MagicMock(spec=ServerSet)
mock_serverset.join = mock.MagicMock()
mock_serverset.join.side_effect = [KazooException("Whoops the ensemble is down!"), "member0001"]
mock_serverset.cancel = mock.MagicMock()
endpoint = Endpoint("localhost", 12345)
clock = ThreadedClock(31337.0)
announcer = Announcer(mock_serverset, endpoint, clock=clock, exception_wait=Amount(2, Time.SECONDS))
announcer.start()
try:
clock.tick(1.0)
assert announcer.disconnected_time() == 1.0
clock.tick(2.0)
assert announcer.disconnected_time() == 0.0, "Announcer should recover after an exception thrown internally."
assert announcer._membership == "member0001"
finally:
announcer.stop()
def test_sampler_base():
class TestSampler(SamplerBase):
def __init__(self, period, clock):
self.count = 0
SamplerBase.__init__(self, period, clock)
def iterate(self):
self.count += 1
test_clock = ThreadedClock()
sampler = TestSampler(Amount(1, Time.SECONDS), clock=test_clock)
sampler.start()
test_clock.tick(0.5)
assert sampler.count == 0
test_clock.tick(0.5)
assert sampler.count == 1
test_clock.tick(5)
assert sampler.count == 6
assert not sampler.is_stopped()
sampler.stop()
# make sure that stopping the sampler short circuits any sampling
test_clock.tick(5)
assert sampler.count == 6
def test_announcer_on_expiration():
joined = threading.Event()
operations = []
def joined_side_effect(*args, **kw):
# 'global' does not work within python nested functions, so we cannot use a
# counter here, so instead we do append/len (see PEP-3104)
operations.append(1)
if len(operations) == 1 or len(operations) == 3:
joined.set()
return 'membership %d' % len(operations)
else:
raise KazooException('Failed to reconnect')
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset.join = MagicMock()
mock_serverset.join.side_effect = joined_side_effect
mock_serverset.cancel = MagicMock()
endpoint = Endpoint('localhost', 12345)
clock = ThreadedClock(31337.0)
announcer = Announcer(
mock_serverset, endpoint, clock=clock, exception_wait=Amount(2, Time.SECONDS))
announcer.start()
try:
joined.wait(timeout=1.0)
assert joined.is_set()
assert announcer._membership == 'membership 1'
assert announcer.disconnected_time() == 0.0
clock.tick(1.0)
assert announcer.disconnected_time() == 0.0
announcer.on_expiration() # expect exception
clock.tick(1.0)
assert announcer.disconnected_time() == 1.0, (
'Announcer should be disconnected on expiration.')
clock.tick(10.0)
assert announcer.disconnected_time() == 0.0, (
'Announcer should not advance disconnection time when connected.')
assert announcer._membership == 'membership 3'
finally:
announcer.stop()
def test_announcer_under_normal_circumstances():
joined = threading.Event()
def joined_side_effect(*args, **kw):
joined.set()
return 'membership foo'
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset.join = MagicMock()
mock_serverset.join.side_effect = joined_side_effect
mock_serverset.cancel = MagicMock()
endpoint = Endpoint('localhost', 12345)
clock = ThreadedClock(31337.0)
announcer = Announcer(mock_serverset, endpoint, clock=clock)
assert announcer.disconnected_time() == 0.0
clock.tick(1.0)
assert announcer.disconnected_time() == 1.0, (
'Announcer should advance disconnection time when not yet initially connected.')
announcer.start()
try:
joined.wait(timeout=1.0)
assert joined.is_set()
assert announcer.disconnected_time() == 0.0
clock.tick(1.0)
assert announcer.disconnected_time() == 0.0, (
'Announcer should not advance disconnection time when connected.')
assert announcer._membership == 'membership foo'
finally:
announcer.stop()
mock_serverset.cancel.assert_called_with('membership foo')
assert announcer.disconnected_time() == 0.0
clock.tick(1.0)
assert announcer.disconnected_time() == 0.0, (
'Announcer should not advance disconnection time when stopped.')
def test_tracing_timed():
sio = Compatibility.StringIO()
clock = ThreadedClock()
final_trace = []
class PrintTraceInterceptor(Tracer):
def print_trace(self, *args, **kw):
final_trace.append(self._local.parent)
tracer = PrintTraceInterceptor(output=sio, clock=clock, predicate=lambda v: False)
assert not hasattr(tracer._local, 'parent')
with tracer.timed('hello'):
clock.tick(1.0)
with tracer.timed('world 1'):
clock.tick(1.0)
with tracer.timed('world 2'):
clock.tick(1.0)
assert len(final_trace) == 1
final_trace = final_trace[0]
assert final_trace._start == 0
assert final_trace._stop == 3
assert final_trace.duration() == 3
assert final_trace.msg == 'hello'
assert len(final_trace.children) == 2
child = final_trace.children[0]
assert child._start == 1
assert child._stop == 2
assert child.parent is final_trace
assert child.msg == 'world 1'
child = final_trace.children[1]
assert child._start == 2
assert child._stop == 3
assert child.parent is final_trace
assert child.msg == 'world 2'
# should not log if verbosity low
assert sio.getvalue() == ''
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health,
interval_secs=5,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, 10)
assert hct.status is None
self._clock.tick(6)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(3)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(5)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=5)
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
# this is an implementation detail -- we healthcheck in the initializer and
# healthcheck in the run loop. if we ever change the implementation, expect
# this to break.
assert self._checker.health.call_count == 2
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
# 2 consecutive health check failures followed by a successful health check.
epsilon = 0.001
self._clock.tick(initial_interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 0
# 3 consecutive health check failures.
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status.status == TaskState.Value('TASK_FAILED')
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 6
@pytest.mark.skipif('True', reason='Flaky test (AURORA-1182)')
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check,
interval_secs=1,
initial_interval_secs=1,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker,
amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker,
amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
def test_not_converged():
clock1 = ThreadedClock(0)
clock2 = ThreadedClock(0)
def run():
clock1.sleep(1)
clock2.sleep(1)
th = threading.Thread(target=run)
th.daemon = True
th.start()
assert clock1.converge(threads=[th])
clock1.assert_waiting(th, 1)
assert clock2.converge(threads=[th], timeout=0.1) is False
clock2.assert_not_waiting(th)
clock1.tick(1)
clock2.tick(2)
clock1.converge(threads=[th])
clock2.converge(threads=[th])
clock1.assert_not_waiting(th)
clock2.assert_not_waiting(th)
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock()
self._mox = mox.Mox()
self._checker = self._mox.CreateMockAnything()
def expect_health_check(self, status, num_calls=1):
for x in range(int(num_calls)):
self._checker().AndReturn((status, 'reason'))
def replay(self):
self._mox.ReplayAll()
def verify(self):
self._mox.VerifyAll()
def test_initial_interval_2x(self):
self.expect_health_check(False)
self.replay()
hct = HealthCheckerThread(self._checker, interval_secs=5, clock=self._clock)
hct.start()
thread_yield()
assert hct.status is None
self._clock.tick(6)
assert hct.status is None
self._clock.tick(3)
assert hct.status is None
self._clock.tick(5)
thread_yield()
assert hct.status is not None
hct.stop()
self.verify()
def test_initial_interval_whatev(self):
self.expect_health_check(False)
self.replay()
hct = HealthCheckerThread(
self._checker,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
assert hct.status is not None
hct.stop()
self.verify()
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.expect_health_check(False, num_calls=2)
self.expect_health_check(True)
self.expect_health_check(False, num_calls=3)
self.replay()
hct = HealthCheckerThread(
self._checker,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
# 2 consecutive health check failures followed by a successful health check.
self._clock.tick(initial_interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
# 3 consecutive health check failures.
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
thread_yield()
assert hct.status is not None
hct.stop()
self.verify()
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.initial_interval_secs = 15
self.interval_secs = 10
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_grace_period_2x_success(self):
'''Grace period is 2 x interval and health checks succeed.'''
self.append_health_checks(True, num_calls=2)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.threaded_health_checker.running is True
hct.stop()
assert self._checker.health.call_count == 1
def test_grace_period_2x_failure_then_success(self):
'''Grace period is 2 x interval and health checks fail then succeed.'''
self.append_health_checks(False)
self.append_health_checks(True)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.threaded_health_checker.running is True
hct.stop()
assert self._checker.health.call_count == 2
def test_grace_period_2x_failure(self):
'''
Grace period is 2 x interval and all health checks fail.
Failures are ignored when in grace period.
'''
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
assert hct.threaded_health_checker.running is False
hct.stop()
assert self._checker.health.call_count == 3
def test_success_outside_grace_period(self):
'''
Health checks fail inside grace period, but pass outside and leads to success
'''
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.threaded_health_checker.running is False
self._clock.tick(self.interval_secs)
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.threaded_health_checker.running is True
hct.stop()
assert self._checker.health.call_count == 3
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(
self._checker.health,
interval_secs=self.interval_secs,
grace_period_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, self.interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
hct.stop()
assert self._checker.health.call_count == 1
def test_consecutive_failures_max_failures(self):
'''Verify that a task is unhealthy after max_consecutive_failures is exceeded'''
grace_period_secs = self.initial_interval_secs
interval_secs = self.interval_secs
self.append_health_checks(True, num_calls=2)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
grace_period_secs=grace_period_secs,
max_consecutive_failures=2,
min_consecutive_successes=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
assert hct.metrics.sample()['consecutive_failures'] == 0
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.metrics.sample()['consecutive_failures'] == 0
assert hct.threaded_health_checker.running is True
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Task is healthy.', TaskState.Value('TASK_RUNNING'))
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 5
def test_consecutive_failures_failfast(self):
'''Verify that health check is failed fast'''
grace_period_secs = self.initial_interval_secs
interval_secs = self.interval_secs
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
grace_period_secs=grace_period_secs,
max_consecutive_failures=2,
min_consecutive_successes=2,
clock=self._clock)
hct.start()
# 3 consecutive health check failures causes fail-fast
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult(None, TaskState.Value('TASK_STARTING'))
# failure is ignored inside grace_period_secs
assert hct.metrics.sample()['consecutive_failures'] == 0
self._clock.tick(interval_secs)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, interval_secs)
assert hct.status == StatusResult('Failed health check! reason', TaskState.Value('TASK_FAILED'))
assert hct.metrics.sample()['consecutive_failures'] == 1
hct.stop()
assert self._checker.health.call_count == 2
@pytest.mark.skipif('True', reason='Flaky test (AURORA-1182)')
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check, interval_secs=1, grace_period_secs=1, clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
def setUp(self): self._clock = ThreadedClock() self._mox = mox.Mox() self._checker = self._mox.CreateMockAnything()
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock()
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.health)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health, interval_secs=5, clock=self._clock)
hct.start()
thread_yield()
assert hct.status is None
self._clock.tick(6)
assert hct.status is None
self._clock.tick(3)
assert hct.status is None
self._clock.tick(5)
thread_yield()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_initial_interval_whatev(self):
self.append_health_checks(False)
hct = HealthChecker(
self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
# 2 consecutive health check failures followed by a successful health check.
self._clock.tick(initial_interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
# 3 consecutive health check failures.
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
thread_yield()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 6
def test_with_events(num_threads):
event = threading.Event()
hits = []
hits_before, hits_after = 0, 0
clock = ThreadedClock(0)
def hit_me():
clock.sleep(0.1)
hits.append(True)
threads = []
for _ in range(num_threads):
th = threading.Thread(target=hit_me)
th.daemon = True
th.start()
threads.append(th)
clock.converge(threads=threads)
for th in threads:
clock.assert_waiting(th, 0.1)
clock.tick(0.05)
clock.converge(threads=threads)
hits_before += len(hits)
with pytest.raises(AssertionError):
clock.assert_waiting(threads[0], 234)
clock.tick(0.05)
clock.converge(threads=threads)
hits_after += len(hits)
for th in threads:
clock.assert_not_waiting(th)
with pytest.raises(AssertionError):
clock.assert_waiting(th, 0.1)
assert hits_before == 0
assert hits_after == num_threads
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock()
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.health)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health, interval_secs=5, clock=self._clock)
hct.start()
thread_yield()
assert hct.status is None
self._clock.tick(6)
assert hct.status is None
self._clock.tick(3)
assert hct.status is None
self._clock.tick(5)
thread_yield()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_initial_interval_whatev(self):
self.append_health_checks(False)
hct = HealthChecker(
self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
# 2 consecutive health check failures followed by a successful health check.
self._clock.tick(initial_interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
# 3 consecutive health check failures.
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
assert hct.status is None
self._clock.tick(interval_secs)
thread_yield()
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 6
class TestHealthChecker(unittest.TestCase):
def setUp(self):
self._clock = ThreadedClock(0)
self._checker = mock.Mock(spec=HttpSignaler)
self.fake_health_checks = []
def mock_health_check():
return self.fake_health_checks.pop(0)
self._checker.health = mock.Mock(spec=self._checker.__call__)
self._checker.health.side_effect = mock_health_check
def append_health_checks(self, status, num_calls=1):
for i in range(num_calls):
self.fake_health_checks.append((status, 'reason'))
def test_initial_interval_2x(self):
self.append_health_checks(False)
hct = HealthChecker(self._checker.health, interval_secs=5, clock=self._clock)
hct.start()
assert self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, 10)
assert hct.status is None
self._clock.tick(6)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(3)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status is None
self._clock.tick(5)
assert self._clock.converge(threads=[hct.threaded_health_checker])
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
assert self._checker.health.call_count == 1
def test_initial_interval_whatev(self):
self.append_health_checks(False, 2)
hct = HealthChecker(
self._checker.health,
interval_secs=5,
initial_interval_secs=0,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=5)
assert hct.status.status == TaskState.Value('TASK_FAILED')
hct.stop()
# this is an implementation detail -- we healthcheck in the initializer and
# healthcheck in the run loop. if we ever change the implementation, expect
# this to break.
assert self._checker.health.call_count == 2
def test_consecutive_failures(self):
'''Verify that a task is unhealthy only after max_consecutive_failures is exceeded'''
initial_interval_secs = 2
interval_secs = 1
self.append_health_checks(False, num_calls=2)
self.append_health_checks(True)
self.append_health_checks(False, num_calls=3)
hct = HealthChecker(
self._checker.health,
interval_secs=interval_secs,
initial_interval_secs=initial_interval_secs,
max_consecutive_failures=2,
clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
# 2 consecutive health check failures followed by a successful health check.
epsilon = 0.001
self._clock.tick(initial_interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 0
# 3 consecutive health check failures.
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 1
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status is None
assert hct.metrics.sample()['consecutive_failures'] == 2
self._clock.tick(interval_secs + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct.status.status == TaskState.Value('TASK_FAILED')
assert hct.metrics.sample()['consecutive_failures'] == 3
hct.stop()
assert self._checker.health.call_count == 6
def test_health_checker_metrics(self):
def slow_check():
self._clock.sleep(0.5)
return (True, None)
hct = HealthChecker(slow_check, interval_secs=1, initial_interval_secs=1, clock=self._clock)
hct.start()
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
# start the health check (during health check it is still 0)
epsilon = 0.001
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=0.5)
assert hct._total_latency == 0
assert hct.metrics.sample()['total_latency_secs'] == 0
assert hct.metrics.sample()['checks'] == 0
# finish the health check
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 0.5
assert hct.metrics.sample()['total_latency_secs'] == 0.5
assert hct.metrics.sample()['checks'] == 1
# tick again
self._clock.tick(1.0 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.tick(0.5 + epsilon)
self._clock.converge(threads=[hct.threaded_health_checker])
self._clock.assert_waiting(hct.threaded_health_checker, amount=1) # interval_secs
assert hct._total_latency == 1.0
assert hct.metrics.sample()['total_latency_secs'] == 1.0
assert hct.metrics.sample()['checks'] == 2
