def test_exception_whitelist(error_func):
breaker = CircuitBreaker(error_threshold=1,
exception_whitelist=[IOError],
recovery_timeout=1)
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
def test_exception_whitelist_supports_inheritance(error_func):
breaker = CircuitBreaker(error_threshold=1,
exception_whitelist=[Exception],
recovery_timeout=1)
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
def test_exception_blacklist_filters_errors(error_func):
# When the blacklist is specified, only those errors are caught
breaker = CircuitBreaker(error_threshold=1,
exception_blacklist=[ValueError],
recovery_timeout=1)
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
def test_breaker_recloses_after_recovery_time(error_func):
breaker = CircuitBreaker(error_threshold=1, recovery_timeout=1)
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.OPEN
sleep(1)
assert breaker.state == CircuitBreakerState.HALF_OPEN
def test_exception_blacklist_supports_inheritance(error_func):
# When the blacklist is specified, only those errors are caught
breaker = CircuitBreaker(error_threshold=1,
exception_blacklist=[Exception],
recovery_timeout=1)
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.OPEN
def test_notifies_on_breaker_open(error_func, io_error):
mock_open = mock.Mock()
breaker = CircuitBreaker(error_threshold=1, on_open=mock_open)
assert mock_open.call_count == 0
with pytest.raises(IOError):
breaker.call(error_func)
mock_open.assert_called_once()
mock_open.assert_called_with(breaker, io_error)
def half_open_breaker(error_func):
breaker = CircuitBreaker(error_threshold=1,
recovery_timeout=1,
recovery_threshold=2)
with pytest.raises(IOError):
breaker.call(error_func)
sleep(1)
return breaker
def test_get_circuits_not_empty():
registry = CircuitBreakerRegistry()
db_circuit = CircuitBreaker(breaker_id="db")
service_circuit = CircuitBreaker(breaker_id="service")
registry.register(db_circuit)
registry.register(service_circuit)
registered = registry.get_circuits()
assert len(registered) == 2
assert db_circuit in registered
assert service_circuit in registered
def test_can_detect_errors_that_are_not_exceptions(error_val, expected_state):
breaker = CircuitBreaker(
detect_error=lambda ret_val: ret_val is False,
error_threshold=1,
)
def return_code_error():
return error_val
result = breaker.call(return_code_error)
assert result is error_val
assert breaker.state is expected_state
def test_get_open_circuits_when_one_circuit_is_open(error_func):
registry = CircuitBreakerRegistry()
open_circuit = CircuitBreaker(breaker_id="open_breaker", error_threshold=1)
closed_circuit = CircuitBreaker(breaker_id="closed_breaker")
registry.register(open_circuit)
registry.register(closed_circuit)
with pytest.raises(IOError):
open_circuit.call(error_func)
opened = registry.get_open_circuits()
assert len(opened) == 1
assert open_circuit in opened
assert closed_circuit not in opened
def test_get_open_circuits_when_all_circuits_are_closed():
registry = CircuitBreakerRegistry()
circuit = CircuitBreaker(breaker_id="closed_breaker", error_threshold=1)
registry.register(circuit)
opened = registry.get_open_circuits()
assert len(opened) == 0
def test_register_new_circuit_breaker():
registry = CircuitBreakerRegistry()
circuit_breaker = CircuitBreaker()
registry.register(circuit_breaker)
registered = registry.get_circuits()
assert len(registered) == 1
assert registered[0] == circuit_breaker
def test_register_existing_circuit_breaker():
registry = CircuitBreakerRegistry()
circuit_breaker = CircuitBreaker()
registry.register(circuit_breaker)
with pytest.raises(CircuitBreakerRegistryException):
registry.register(circuit_breaker)
registered = registry.get_circuits()
assert len(registered) == 1
assert registered[0] == circuit_breaker
def test_calling_an_open_breaker_raises_an_error(error_func):
breaker = CircuitBreaker(error_threshold=1,
recovery_timeout=1,
recovery_threshold=2)
with pytest.raises(IOError):
breaker.call(error_func)
with pytest.raises(CircuitBreakerException):
breaker.call(error_func)
def test_breaker_opens_after_specified_number_of_errors(error_func):
breaker = CircuitBreaker(error_threshold=2)
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.OPEN
def test_net_error_strategy_half_open_to_closed(error_func, success_func):
breaker = CircuitBreaker(
strategy=CircuitBreakerStrategy.NET_ERROR,
error_threshold=1,
recovery_timeout=1,
)
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.OPEN
sleep(1)
assert breaker.state == CircuitBreakerState.HALF_OPEN
breaker.call(success_func)
assert breaker.state == CircuitBreakerState.CLOSED
def test_notifies_on_breaker_close(error_func, success_func):
mock_close = mock.Mock()
breaker = CircuitBreaker(
error_threshold=1,
on_close=mock_close,
recovery_timeout=1,
)
assert mock_close.call_count == 0
with pytest.raises(IOError):
breaker.call(error_func)
assert mock_close.call_count == 0
sleep(1)
breaker.call(success_func)
assert mock_close.call_count == 1
def test_breaker_starts_closed():
breaker = CircuitBreaker()
assert breaker.state == CircuitBreakerState.CLOSED
def test_unknown_strategy_causes_error():
with pytest.raises(ValueError):
CircuitBreaker(strategy="foo")
def test_net_error_strategy_count_never_negative(success_func):
breaker = CircuitBreaker(strategy=CircuitBreakerStrategy.NET_ERROR, )
breaker.call(success_func)
assert breaker._strategy._net_error_count == 0
def test_net_error_strategy(error_func, success_func):
breaker = CircuitBreaker(
strategy=CircuitBreakerStrategy.NET_ERROR,
error_threshold=3,
)
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
breaker.call(success_func)
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.CLOSED
with pytest.raises(IOError):
breaker.call(error_func)
assert breaker.state == CircuitBreakerState.OPEN
def test_circuit_breaker_exception_serialization():
breaker = CircuitBreaker(breaker_id="FOO")
exception = CircuitBreakerException(breaker)
exception_str = str(exception)
assert "Circuit FOO OPEN until" in exception_str