class CircuitBreakerThreadsTestCase(unittest.TestCase): """ Tests to reproduce common synchronization errors on CircuitBreaker class. """ def setUp(self): self.breaker = CircuitBreaker(fail_max=3000, reset_timeout=600) def _start_threads(self, target, n): """ Starts `n` threads that calls `target` and waits for them to finish. """ threads = [threading.Thread(target=target) for i in range(n)] [t.start() for t in threads] [t.join() for t in threads] def _mock_function(self, obj, func): """ Replaces a bounded function in `self.breaker` by another. """ setattr(obj, func.__name__, MethodType(func, self.breaker)) def test_fail_thread_safety_context_manager(self): """CircuitBreaker: it should compute a failed call atomically to avoid race conditions, even when using context manager """ def trigger_error(): for n in range(500): try: with self.breaker.context(): raise Exception() except: pass def _inc_counter(self): c = self._fail_counter sleep(0.00005) self._fail_counter = c + 1 self._mock_function(self.breaker, _inc_counter) self._start_threads(trigger_error, 3) self.assertEqual(1500, self.breaker.fail_counter) def test_fail_thread_safety(self): """CircuitBreaker: it should compute a failed call atomically to avoid race conditions. """ @self.breaker def err(): raise Exception() def trigger_error(): for n in range(500): try: err() except: pass def _inc_counter(self): c = self._fail_counter sleep(0.00005) self._fail_counter = c + 1 self._mock_function(self.breaker, _inc_counter) self._start_threads(trigger_error, 3) self.assertEqual(1500, self.breaker.fail_counter) def test_success_thread_safety(self): """CircuitBreaker: it should compute a successful call atomically to avoid race conditions. """ @self.breaker def suc(): return True def trigger_success(): for n in range(500): suc() class SuccessListener(CircuitBreakerListener): def success(self, cb): c = 0 if hasattr(cb, '_success_counter'): c = cb._success_counter sleep(0.00005) cb._success_counter = c + 1 self.breaker.listeners.append(SuccessListener()) self._start_threads(trigger_success, 3) self.assertEqual(1500, self.breaker._success_counter) def test_half_open_thread_safety(self): """CircuitBreaker: it should allow only one trial call when the circuit is half-open. """ self.breaker.half_open() @self.breaker def err(): raise Exception() def trigger_failure(): try: err() except: pass class StateListener(CircuitBreakerListener): def __init__(self): self._count = 0 def before_call(self, cb, fun, *args, **kwargs): sleep(0.00005) def state_change(self, cb, old_state, new_state): if new_state == 'half_open': self._count += 1 state_listener = StateListener() self.breaker.listeners.append(state_listener) self._start_threads(trigger_failure, 5) self.assertEqual(1, state_listener._count) def test_fail_max_thread_safety(self): """CircuitBreaker: it should not allow more failed calls than 'fail_max' setting. """ @self.breaker def err(): raise Exception() def trigger_error(): for i in range(2000): try: err() except: pass class SleepListener(CircuitBreakerListener): def before_call(self, cb, func, *args, **kwargs): sleep(0.00005) self.breaker.listeners.append(SleepListener()) self._start_threads(trigger_error, 3) self.assertEqual(self.breaker.fail_max, self.breaker.fail_counter)