def test_basic_retry(self): succeeding_operation = create_succeeding_operation() policy = RetryPolicy() loop.run_until_complete( Failsafe(retry_policy=policy).run(succeeding_operation) ) assert succeeding_operation.called == 1
def __init__(self, url, auth=None, session=None, trailing_slash=False, retries=6, autopaginate=None): if auth is not None and not isinstance(auth, aiohttp.BasicAuth): auth = aiohttp.BasicAuth(*auth) super(GenericClient, self).__init__(url, auth, session, trailing_slash, autopaginate) max_failures = retries - 1 circuit_breaker = CircuitBreaker(maximum_failures=max_failures) retry_policy = RetryPolicy( allowed_retries=retries, retriable_exceptions=[ClientConnectionError], abortable_exceptions=[ exceptions.BadRequestError, exceptions.NotAuthenticatedError, exceptions.ResourceNotFound, exceptions.MultipleResourcesFound, exceptions.HTTPError, ValueError, ], ) self.failsafe = Failsafe(circuit_breaker=circuit_breaker, retry_policy=retry_policy)
def __init__(self): retry_policy = RetryPolicy(allowed_retries=4) self.failsafe = Failsafe(retry_policy=retry_policy, circuit_breaker=CircuitBreaker(maximum_failures=8)) self.failsafe_fallback = Failsafe(retry_policy=retry_policy, circuit_breaker=CircuitBreaker(maximum_failures=8))
def test_circuit_breaker_with_retries(self): failing_operation = create_failing_operation() with pytest.raises(CircuitOpen): policy = RetryPolicy(5, [SomeRetriableException]) circuit_breaker = CircuitBreaker(maximum_failures=2) loop.run_until_complete( Failsafe( retry_policy=policy, circuit_breaker=circuit_breaker).run(failing_operation)) assert failing_operation.called == 2
def test_original_exception_is_raised_and_fallback_is_not_executed_on_abortion(self): async def call(fallback_option): assert fallback_option == "fallback option1" raise ValueError() policy = RetryPolicy(abortable_exceptions=[ValueError]) fallback_failsafe = FallbackFailsafe(["fallback option1", "fallback option2"], retry_policy_factory=lambda _: policy) with pytest.raises(ValueError): loop.run_until_complete( fallback_failsafe.run(call))
def test_retry_on_custom_exception(self): failing_operation = create_failing_operation() retries = 3 policy = RetryPolicy(retries, [SomeRetriableException]) failsafe = Failsafe(retry_policy=policy) assert failing_operation.called == 0 with pytest.raises(RetriesExhausted): loop.run_until_complete(failsafe.run(failing_operation)) assert failing_operation.called == retries + 1
def test_retry_four_times(self): failing_operation = create_failing_operation() expected_attempts = 5 retries = 4 policy = RetryPolicy(retries) failsafe = Failsafe(retry_policy=policy) assert failing_operation.called == 0 with pytest.raises(RetriesExhausted): loop.run_until_complete(failsafe.run(failing_operation)) assert failing_operation.called == expected_attempts
def test_delay(self): failing_operation = create_failing_operation() retries = 3 delay = Delay(timedelta(seconds=0.2)) policy = RetryPolicy(retries, [SomeRetriableException], backoff=delay) with pytest.raises(RetriesExhausted): loop.run_until_complete( Failsafe(retry_policy=policy).run(failing_operation)) assert asyncio.sleep.mock_calls == [ call(0.2), call(0.2), call(0.2), ]
def test_circuit_breaker_circuitopener_raises_circuitopen_with_cause(self): original_exception = SomeRetriableException("My Error Message") failing_operation = create_failing_operation(original_exception) try: policy = RetryPolicy(5, [SomeRetriableException]) circuit_breaker = CircuitBreaker(maximum_failures=2) loop.run_until_complete( Failsafe( retry_policy=policy, circuit_breaker=circuit_breaker).run(failing_operation)) raise Exception("Expected CircuitOpen exception") except CircuitOpen as e: assert e.__cause__ is original_exception
def test_circuit_breaker_with_abort(self): aborting_operation = create_aborting_operation() policy = RetryPolicy(abortable_exceptions=[SomeAbortableException]) circuit_breaker = CircuitBreaker(maximum_failures=2) circuit_breaker.record_failure = MagicMock() with pytest.raises(SomeAbortableException): loop.run_until_complete( Failsafe( retry_policy=policy, circuit_breaker=circuit_breaker).run(aborting_operation)) circuit_breaker.record_failure.assert_not_called() assert aborting_operation.called == 1
def test_circuit_breaker_opened_circuit_has_no_cause(self): failing_operation = create_failing_operation() try: policy = RetryPolicy(5, [SomeRetriableException]) circuit_breaker = CircuitBreaker(maximum_failures=2) circuit_breaker.open() loop.run_until_complete( Failsafe( retry_policy=policy, circuit_breaker=circuit_breaker).run(failing_operation)) raise Exception("Expected CircuitOpen exception") except CircuitOpen as e: assert e.__cause__ is None assert failing_operation.called == 0
def __init__(self, fallback_options, retry_policy_factory=None, circuit_breaker_factory=None): """ :param fallback_options: a list of objects which will differentiate between different fallback calls. An item from this list will be passed as the first parameter to the function provided to the run method. :param retry_policy_factory: factory function accepting a fallback option and returning a retry policy :param circuit_breaker_factory: factory function accepting a fallback option and returning a circuit breaker """ retry_policy_factory = retry_policy_factory or (lambda _: RetryPolicy()) circuit_breaker_factory = circuit_breaker_factory or (lambda _: CircuitBreaker()) def _create_failsafe(option): return Failsafe(retry_policy=retry_policy_factory(option), circuit_breaker=circuit_breaker_factory(option)) self.failsafes = [(option, _create_failsafe(option)) for option in fallback_options]
def __init__(self): self.failsafe = Failsafe( retry_policy=RetryPolicy(allowed_retries=4, abortable_exceptions=[NotFoundError]), circuit_breaker=CircuitBreaker(maximum_failures=8))
import aiohttp from failsafe import CircuitBreaker, Failsafe, FailsafeError, RetryPolicy from sanic import Sanic from sanic.response import json, text from sanic_prometheus import monitor app = Sanic() circuit = CircuitBreaker( maximum_failures=3, reset_timeout_seconds=10 ) failsafe = Failsafe( circuit_breaker=circuit, retry_policy=RetryPolicy() ) @app.route("/") async def index(request): return text("Hello world!") @app.route("/health") async def health(request): if circuit.current_state == "open": return text("Unavailable", status=503) return text("OK")