def test_basic_retry(self):
succeeding_operation = create_succeeding_operation()
policy = RetryPolicy()
loop.run_until_complete(
Failsafe(retry_policy=policy).run(succeeding_operation)
)
assert succeeding_operation.called == 1
def __init__(self,
url,
auth=None,
session=None,
trailing_slash=False,
retries=6,
autopaginate=None):
if auth is not None and not isinstance(auth, aiohttp.BasicAuth):
auth = aiohttp.BasicAuth(*auth)
super(GenericClient, self).__init__(url, auth, session, trailing_slash,
autopaginate)
max_failures = retries - 1
circuit_breaker = CircuitBreaker(maximum_failures=max_failures)
retry_policy = RetryPolicy(
allowed_retries=retries,
retriable_exceptions=[ClientConnectionError],
abortable_exceptions=[
exceptions.BadRequestError,
exceptions.NotAuthenticatedError,
exceptions.ResourceNotFound,
exceptions.MultipleResourcesFound,
exceptions.HTTPError,
ValueError,
],
)
self.failsafe = Failsafe(circuit_breaker=circuit_breaker,
retry_policy=retry_policy)
def __init__(self):
retry_policy = RetryPolicy(allowed_retries=4)
self.failsafe = Failsafe(retry_policy=retry_policy,
circuit_breaker=CircuitBreaker(maximum_failures=8))
self.failsafe_fallback = Failsafe(retry_policy=retry_policy,
circuit_breaker=CircuitBreaker(maximum_failures=8))
def test_circuit_breaker_with_retries(self):
failing_operation = create_failing_operation()
with pytest.raises(CircuitOpen):
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
assert failing_operation.called == 2
def test_original_exception_is_raised_and_fallback_is_not_executed_on_abortion(self):
async def call(fallback_option):
assert fallback_option == "fallback option1"
raise ValueError()
policy = RetryPolicy(abortable_exceptions=[ValueError])
fallback_failsafe = FallbackFailsafe(["fallback option1", "fallback option2"],
retry_policy_factory=lambda _: policy)
with pytest.raises(ValueError):
loop.run_until_complete(
fallback_failsafe.run(call))
def test_retry_on_custom_exception(self):
failing_operation = create_failing_operation()
retries = 3
policy = RetryPolicy(retries, [SomeRetriableException])
failsafe = Failsafe(retry_policy=policy)
assert failing_operation.called == 0
with pytest.raises(RetriesExhausted):
loop.run_until_complete(failsafe.run(failing_operation))
assert failing_operation.called == retries + 1
def test_retry_four_times(self):
failing_operation = create_failing_operation()
expected_attempts = 5
retries = 4
policy = RetryPolicy(retries)
failsafe = Failsafe(retry_policy=policy)
assert failing_operation.called == 0
with pytest.raises(RetriesExhausted):
loop.run_until_complete(failsafe.run(failing_operation))
assert failing_operation.called == expected_attempts
def test_delay(self):
failing_operation = create_failing_operation()
retries = 3
delay = Delay(timedelta(seconds=0.2))
policy = RetryPolicy(retries, [SomeRetriableException], backoff=delay)
with pytest.raises(RetriesExhausted):
loop.run_until_complete(
Failsafe(retry_policy=policy).run(failing_operation))
assert asyncio.sleep.mock_calls == [
call(0.2),
call(0.2),
call(0.2),
]
def test_circuit_breaker_circuitopener_raises_circuitopen_with_cause(self):
original_exception = SomeRetriableException("My Error Message")
failing_operation = create_failing_operation(original_exception)
try:
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
raise Exception("Expected CircuitOpen exception")
except CircuitOpen as e:
assert e.__cause__ is original_exception
def test_circuit_breaker_with_abort(self):
aborting_operation = create_aborting_operation()
policy = RetryPolicy(abortable_exceptions=[SomeAbortableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
circuit_breaker.record_failure = MagicMock()
with pytest.raises(SomeAbortableException):
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(aborting_operation))
circuit_breaker.record_failure.assert_not_called()
assert aborting_operation.called == 1
def test_circuit_breaker_opened_circuit_has_no_cause(self):
failing_operation = create_failing_operation()
try:
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
circuit_breaker.open()
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
raise Exception("Expected CircuitOpen exception")
except CircuitOpen as e:
assert e.__cause__ is None
assert failing_operation.called == 0
def __init__(self, fallback_options, retry_policy_factory=None, circuit_breaker_factory=None):
"""
:param fallback_options: a list of objects which will differentiate between different fallback calls. An item
from this list will be passed as the first parameter to the function provided to the run method.
:param retry_policy_factory: factory function accepting a fallback option
and returning a retry policy
:param circuit_breaker_factory: factory function accepting a fallback option
and returning a circuit breaker
"""
retry_policy_factory = retry_policy_factory or (lambda _: RetryPolicy())
circuit_breaker_factory = circuit_breaker_factory or (lambda _: CircuitBreaker())
def _create_failsafe(option):
return Failsafe(retry_policy=retry_policy_factory(option),
circuit_breaker=circuit_breaker_factory(option))
self.failsafes = [(option, _create_failsafe(option))
for option in fallback_options]
def __init__(self):
self.failsafe = Failsafe(
retry_policy=RetryPolicy(allowed_retries=4,
abortable_exceptions=[NotFoundError]),
circuit_breaker=CircuitBreaker(maximum_failures=8))
import aiohttp
from failsafe import CircuitBreaker, Failsafe, FailsafeError, RetryPolicy
from sanic import Sanic
from sanic.response import json, text
from sanic_prometheus import monitor
app = Sanic()
circuit = CircuitBreaker(
maximum_failures=3,
reset_timeout_seconds=10
)
failsafe = Failsafe(
circuit_breaker=circuit,
retry_policy=RetryPolicy()
)
@app.route("/")
async def index(request):
return text("Hello world!")
@app.route("/health")
async def health(request):
if circuit.current_state == "open":
return text("Unavailable", status=503)
return text("OK")
