def test_fail_max_thread_safety(self):
"""CircuitBreaker: it should not allow more failed calls than 'fail_max'
setting. Note that with Redis, where we have separate systems
incrementing the counter, we can get concurrent updates such that the
counter is greater than the 'fail_max' by the number of systems. To
prevent this, we'd need to take out a lock amongst all systems before
trying the call.
"""
breaker = CircuitBreaker()
@breaker
def err():
raise DummyException()
def trigger_error():
for i in range(2000):
try:
err()
except DummyException:
pass
except CircuitBreakerError:
pass
class SleepListener(CircuitBreakerListener):
def before_call(self, breaker, func, *args, **kwargs):
sleep(0.00005)
breaker.add_listener(SleepListener())
num_threads = 3
self._start_threads(trigger_error, num_threads)
self.assertTrue(breaker.fail_counter < breaker.fail_max + num_threads)
def test_success_thread_safety(self):
"""It should compute a successful call atomically to avoid race conditions."""
breaker = CircuitBreaker(fail_max=3000,
timeout_duration=timedelta(seconds=1),
state_storage=self.storage)
@breaker
def suc():
return True
def trigger_success():
for n in range(500):
suc()
class SuccessListener(CircuitBreakerListener):
def success(self, breaker):
c = 0
if hasattr(breaker, '_success_counter'):
c = breaker._success_counter
sleep(0.00005)
breaker._success_counter = c + 1
breaker.add_listener(SuccessListener())
self._start_threads(trigger_success, 3)
self.assertEqual(1500, breaker._success_counter)
def test_remove_listener():
""" it should allow the user to remove a listener."""
breaker = CircuitBreaker()
first = CircuitBreakerListener()
breaker.add_listener(first)
assert (first, ) == breaker.listeners
breaker.remove_listener(first)
assert () == breaker.listeners
def test_add_listener():
""" It should allow the user to add a listener at a later time."""
breaker = CircuitBreaker()
assert () == breaker.listeners
first = CircuitBreakerListener()
breaker.add_listener(first)
assert (first, ) == breaker.listeners
second = CircuitBreakerListener()
breaker.add_listener(second)
assert (first, second) == breaker.listeners
def test_half_open_thread_safety(self):
"""
it should allow only one trial call when the circuit is half-open.
"""
breaker = CircuitBreaker(fail_max=1,
timeout_duration=timedelta(seconds=0.01))
breaker.open()
sleep(0.01)
@breaker
def err():
raise DummyException()
def trigger_failure():
try:
err()
except DummyException:
pass
except CircuitBreakerError:
pass
class StateListener(CircuitBreakerListener):
def __init__(self):
self._count = 0
def before_call(self, breaker, func, *args, **kwargs):
sleep(0.00005)
def state_change(self, breaker, old, new):
if new.name == STATE_HALF_OPEN:
self._count += 1
state_listener = StateListener()
breaker.add_listener(state_listener)
self._start_threads(trigger_failure, 5)
self.assertEqual(1, state_listener._count)
