def test_fail_max_thread_safety(self): """CircuitBreaker: it should not allow more failed calls than 'fail_max' setting. Note that with Redis, where we have separate systems incrementing the counter, we can get concurrent updates such that the counter is greater than the 'fail_max' by the number of systems. To prevent this, we'd need to take out a lock amongst all systems before trying the call. """ breaker = CircuitBreaker() @breaker def err(): raise DummyException() def trigger_error(): for i in range(2000): try: err() except DummyException: pass except CircuitBreakerError: pass class SleepListener(CircuitBreakerListener): def before_call(self, breaker, func, *args, **kwargs): sleep(0.00005) breaker.add_listener(SleepListener()) num_threads = 3 self._start_threads(trigger_error, num_threads) self.assertTrue(breaker.fail_counter < breaker.fail_max + num_threads)
def test_success_thread_safety(self): """It should compute a successful call atomically to avoid race conditions.""" breaker = CircuitBreaker(fail_max=3000, timeout_duration=timedelta(seconds=1), state_storage=self.storage) @breaker def suc(): return True def trigger_success(): for n in range(500): suc() class SuccessListener(CircuitBreakerListener): def success(self, breaker): c = 0 if hasattr(breaker, '_success_counter'): c = breaker._success_counter sleep(0.00005) breaker._success_counter = c + 1 breaker.add_listener(SuccessListener()) self._start_threads(trigger_success, 3) self.assertEqual(1500, breaker._success_counter)
def test_remove_listener(): """ it should allow the user to remove a listener.""" breaker = CircuitBreaker() first = CircuitBreakerListener() breaker.add_listener(first) assert (first, ) == breaker.listeners breaker.remove_listener(first) assert () == breaker.listeners
def test_add_listener(): """ It should allow the user to add a listener at a later time.""" breaker = CircuitBreaker() assert () == breaker.listeners first = CircuitBreakerListener() breaker.add_listener(first) assert (first, ) == breaker.listeners second = CircuitBreakerListener() breaker.add_listener(second) assert (first, second) == breaker.listeners
def test_half_open_thread_safety(self): """ it should allow only one trial call when the circuit is half-open. """ breaker = CircuitBreaker(fail_max=1, timeout_duration=timedelta(seconds=0.01)) breaker.open() sleep(0.01) @breaker def err(): raise DummyException() def trigger_failure(): try: err() except DummyException: pass except CircuitBreakerError: pass class StateListener(CircuitBreakerListener): def __init__(self): self._count = 0 def before_call(self, breaker, func, *args, **kwargs): sleep(0.00005) def state_change(self, breaker, old, new): if new.name == STATE_HALF_OPEN: self._count += 1 state_listener = StateListener() breaker.add_listener(state_listener) self._start_threads(trigger_failure, 5) self.assertEqual(1, state_listener._count)