def test_call_events(storage):
"""It should call the appropriate functions on every successful/failed call.
"""
class Listener(CircuitBreakerListener):
def __init__(self):
self.out = []
def before_call(self, breaker, func, *args, **kwargs):
assert breaker
self.out.append("CALL")
def success(self, breaker):
assert breaker
self.out.append("SUCCESS")
def failure(self, breaker, exception):
assert breaker
assert isinstance(exception, DummyException)
self.out.append("FAILURE")
listener = Listener()
breaker = CircuitBreaker(listeners=(listener, ), state_storage=storage)
assert breaker.call(func_succeed)
with raises(DummyException):
breaker.call(func_exception)
assert ["CALL", "SUCCESS", "CALL", "FAILURE"] == listener.out
def test_one_failed_call(storage):
"""It should keep the circuit closed after a few failures."""
breaker = CircuitBreaker(state_storage=storage)
with raises(DummyException):
breaker.call(func_exception)
assert 1 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
def test_successful_after_timeout(storage, delta):
"""It should close the circuit when a call succeeds after timeout."""
breaker = CircuitBreaker(fail_max=3,
timeout_duration=delta,
state_storage=storage)
func_succeed = func_succeed_counted()
with raises(DummyException):
breaker.call(func_exception)
with raises(DummyException):
breaker.call(func_exception)
assert CircuitBreakerState.CLOSED == breaker.current_state
# Circuit should open
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert CircuitBreakerState.OPEN == breaker.current_state
with raises(CircuitBreakerError):
breaker.call(func_succeed)
assert 3 == breaker.fail_counter
# Wait for timeout, at least a second since redis rounds to a second
sleep(delta.total_seconds() * 2)
# Circuit should close again
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
assert 1 == func_succeed.call_count
def test_one_successful_call_after_failed_call(storage):
"""It should keep the circuit closed after few mixed outcomes."""
breaker = CircuitBreaker(state_storage=storage)
with raises(DummyException):
breaker.call(func_exception)
assert 1 == breaker.fail_counter
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
def test_failed_call_after_timeout(storage, delta):
"""It should half-open the circuit after timeout and close immediately on fail."""
breaker = CircuitBreaker(fail_max=3,
timeout_duration=delta,
state_storage=storage)
with raises(DummyException):
breaker.call(func_exception)
with raises(DummyException):
breaker.call(func_exception)
assert CircuitBreakerState.CLOSED == breaker.current_state
# Circuit should open
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert 3 == breaker.fail_counter
sleep(delta.total_seconds() * 2)
# Circuit should open again
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert 4 == breaker.fail_counter
assert CircuitBreakerState.OPEN == breaker.current_state
def test_failed_call_when_half_open(storage):
"""It should open the circuit when a call fails in half-open state."""
breaker = CircuitBreaker(state_storage=storage)
breaker.half_open()
assert 0 == breaker.fail_counter
assert CircuitBreakerState.HALF_OPEN == breaker.current_state
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert 1 == breaker.fail_counter
assert CircuitBreakerState.OPEN == breaker.current_state
def test_excluded_exceptions():
"""CircuitBreaker: it should ignore specific exceptions.
"""
breaker = CircuitBreaker(exclude=[LookupError])
def err_1():
raise DummyException()
def err_2():
raise LookupError()
def err_3():
raise KeyError()
with raises(DummyException):
breaker.call(err_1)
assert 1 == breaker.fail_counter
# LookupError is not considered a system error
with raises(LookupError):
breaker.call(err_2)
assert 0 == breaker.fail_counter
with raises(DummyException):
breaker.call(err_1)
assert 1 == breaker.fail_counter
# Should consider subclasses as well (KeyError is a subclass of
# LookupError)
with raises(KeyError):
breaker.call(err_3)
assert 0 == breaker.fail_counter
def test_successful_call(storage):
"""It should keep the circuit closed after a successful call."""
breaker = CircuitBreaker(state_storage=storage)
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
def test_double_count():
"""It should not trigger twice if you call CircuitBreaker#call on a decorated function."""
breaker = CircuitBreaker()
@breaker
def err():
"""Docstring"""
raise DummyException()
assert 0 == breaker.fail_counter
with raises(DummyException):
breaker.call(err)
assert 1 == breaker.fail_counter
def test_call_with_args():
""" It should be able to invoke functions with args."""
def func(arg1, arg2):
return arg1, arg2
breaker = CircuitBreaker()
assert (42, 'abc') == breaker.call(func, 42, 'abc')
def test_call_with_kwargs():
""" It should be able to invoke functions with kwargs."""
def func(**kwargs):
return kwargs
breaker = CircuitBreaker()
kwargs = {'a': 1, 'b': 2}
assert kwargs == breaker.call(func, **kwargs)
def test_successful_call_when_half_open(storage):
"""It should close the circuit when a call succeeds in half-open state."""
breaker = CircuitBreaker(state_storage=storage)
breaker.half_open()
assert 0 == breaker.fail_counter
assert CircuitBreakerState.HALF_OPEN == breaker.current_state
# Circuit should open
assert breaker.call(func_succeed)
assert 0 == breaker.fail_counter
assert CircuitBreakerState.CLOSED == breaker.current_state
def test_excluded_exceptions():
"""CircuitBreaker: it should ignore specific exceptions.
"""
breaker = CircuitBreaker(exclude=[
LookupError, lambda e: type(e) == DummyException and e.val == 3
])
def err_1():
raise LookupError()
def err_2():
raise DummyException()
def err_3():
raise KeyError()
def err_4():
raise DummyException(val=3)
# LookupError is not considered a system error
with raises(LookupError):
breaker.call(err_1)
assert 0 == breaker.fail_counter
with raises(DummyException):
breaker.call(err_2)
assert 1 == breaker.fail_counter
# Should consider subclasses as well (KeyError is a subclass of
# LookupError)
with raises(KeyError):
breaker.call(err_3)
assert 0 == breaker.fail_counter
# should filter based on functions as well
with raises(DummyException):
breaker.call(err_4)
assert 0 == breaker.fail_counter
def test_several_failed_calls(storage):
"""It should open the circuit after multiple failures."""
breaker = CircuitBreaker(state_storage=storage, fail_max=3)
with raises(DummyException):
breaker.call(func_exception)
with raises(DummyException):
breaker.call(func_exception)
# Circuit should open
with raises(CircuitBreakerError):
breaker.call(func_exception)
assert breaker.fail_counter == 3
assert breaker.current_state == CircuitBreakerState.OPEN
def test_call_with_no_args():
""" It should be able to invoke functions with no-args."""
breaker = CircuitBreaker()
assert breaker.call(func_succeed)