def __init__(self):
retry_policy = RetryPolicy(allowed_retries=4)
self.failsafe = Failsafe(retry_policy=retry_policy,
circuit_breaker=CircuitBreaker(maximum_failures=8))
self.failsafe_fallback = Failsafe(retry_policy=retry_policy,
circuit_breaker=CircuitBreaker(maximum_failures=8))
def test_retry_on_custom_exception(self):
failing_operation = create_failing_operation()
retries = 3
policy = RetryPolicy(retries, [SomeRetriableException])
failsafe = Failsafe(retry_policy=policy)
assert failing_operation.called == 0
with pytest.raises(RetriesExhausted):
loop.run_until_complete(failsafe.run(failing_operation))
assert failing_operation.called == retries + 1
def test_retry_four_times(self):
failing_operation = create_failing_operation()
expected_attempts = 5
retries = 4
policy = RetryPolicy(retries)
failsafe = Failsafe(retry_policy=policy)
assert failing_operation.called == 0
with pytest.raises(RetriesExhausted):
loop.run_until_complete(failsafe.run(failing_operation))
assert failing_operation.called == expected_attempts
def test_backoff(self):
failing_operation = create_failing_operation()
retries = 3
backoff = Backoff(timedelta(seconds=0.2), timedelta(seconds=1))
policy = RetryPolicy(retries, [SomeRetriableException],
backoff=backoff)
Failsafe(retry_policy=policy)
with pytest.raises(RetriesExhausted):
loop.run_until_complete(
Failsafe(retry_policy=policy).run(failing_operation))
assert asyncio.sleep.mock_calls == [
call(0.2),
call(0.4),
call(0.8),
]
def test_basic_retry(self):
succeeding_operation = create_succeeding_operation()
policy = RetryPolicy()
loop.run_until_complete(
Failsafe(retry_policy=policy).run(succeeding_operation)
)
assert succeeding_operation.called == 1
def __init__(self,
url,
auth=None,
session=None,
trailing_slash=False,
retries=6,
autopaginate=None):
if auth is not None and not isinstance(auth, aiohttp.BasicAuth):
auth = aiohttp.BasicAuth(*auth)
super(GenericClient, self).__init__(url, auth, session, trailing_slash,
autopaginate)
max_failures = retries - 1
circuit_breaker = CircuitBreaker(maximum_failures=max_failures)
retry_policy = RetryPolicy(
allowed_retries=retries,
retriable_exceptions=[ClientConnectionError],
abortable_exceptions=[
exceptions.BadRequestError,
exceptions.NotAuthenticatedError,
exceptions.ResourceNotFound,
exceptions.MultipleResourcesFound,
exceptions.HTTPError,
ValueError,
],
)
self.failsafe = Failsafe(circuit_breaker=circuit_breaker,
retry_policy=retry_policy)
def test_circuit_breaker_with_retries(self):
failing_operation = create_failing_operation()
with pytest.raises(CircuitOpen):
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
assert failing_operation.called == 2
def test_failsafe_on_method_with_arguments(self):
class Operation:
async def task_with_arguments(self, x, y=0):
result = x + y
return "{0}_{1}".format(self.__class__.__name__, result)
operation = Operation()
obtained = loop.run_until_complete(
Failsafe().run(operation.task_with_arguments, 41, y=1)
)
assert obtained == "{0}_42".format(Operation.__name__)
def test_circuit_breaker_circuitopener_raises_circuitopen_with_cause(self):
original_exception = SomeRetriableException("My Error Message")
failing_operation = create_failing_operation(original_exception)
try:
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
raise Exception("Expected CircuitOpen exception")
except CircuitOpen as e:
assert e.__cause__ is original_exception
def test_circuit_breaker_with_abort(self):
aborting_operation = create_aborting_operation()
policy = RetryPolicy(abortable_exceptions=[SomeAbortableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
circuit_breaker.record_failure = MagicMock()
with pytest.raises(SomeAbortableException):
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(aborting_operation))
circuit_breaker.record_failure.assert_not_called()
assert aborting_operation.called == 1
def test_circuit_breaker_opened_circuit_has_no_cause(self):
failing_operation = create_failing_operation()
try:
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
circuit_breaker.open()
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
raise Exception("Expected CircuitOpen exception")
except CircuitOpen as e:
assert e.__cause__ is None
assert failing_operation.called == 0
def __init__(self):
self.failsafe = Failsafe(
retry_policy=RetryPolicy(allowed_retries=4,
abortable_exceptions=[NotFoundError]),
circuit_breaker=CircuitBreaker(maximum_failures=8))
from .utils import apply_filters, remove_script_tags, remove_meta_fragment_tag, is_yesish
logger = logging.getLogger(__name__)
executor = ThreadPoolExecutor(max_workers=cpu_count() * 5)
HTML_FILTERS: Tuple[Callable[[str], str]] = (remove_script_tags,
remove_meta_fragment_tag)
ALLOWED_DOMAINS: Set = set(
dm.strip() for dm in os.getenv('ALLOWED_DOMAINS', '').split(',')
if dm.strip())
CACHE_LIVE_TIME: int = int(os.getenv('CACHE_LIVE_TIME', 3600))
SENTRY_DSN: Optional[str] = os.getenv('SENTRY_DSN')
_ENABLE_CB = is_yesish(os.getenv('ENABLE_CIRCUIT_BREAKER', '0'))
_CB_FAIL_MAX: int = int(os.getenv('CIRCUIT_BREAKER_FAIL_MAX', 5))
_CB_RESET_TIMEOUT: int = int(os.getenv('CIRCUIT_BREAKER_RESET_TIMEOUT', 60))
_BREAKERS = defaultdict(lambda: Failsafe(circuit_breaker=CircuitBreaker(
maximum_failures=_CB_FAIL_MAX, reset_timeout_seconds=_CB_RESET_TIMEOUT)))
if SENTRY_DSN:
sentry = raven.Client(
SENTRY_DSN,
transport=AioHttpTransport,
release=raven.fetch_package_version('prerender'),
site='Prerender',
)
else:
sentry = None
def _save_to_cache(key: str, data: bytes, format: str = 'html') -> None:
try:
cache.set(key, data, CACHE_LIVE_TIME, format)
def test_no_retry(self):
succeeding_operation = create_succeeding_operation()
loop.run_until_complete(Failsafe().run(succeeding_operation))
def _create_failsafe(option):
return Failsafe(retry_policy=retry_policy_factory(option),
circuit_breaker=circuit_breaker_factory(option))
import os
import aiohttp
from failsafe import CircuitBreaker, Failsafe, FailsafeError, RetryPolicy
from sanic import Sanic
from sanic.response import json, text
from sanic_prometheus import monitor
app = Sanic()
circuit = CircuitBreaker(
maximum_failures=3,
reset_timeout_seconds=10
)
failsafe = Failsafe(
circuit_breaker=circuit,
retry_policy=RetryPolicy()
)
@app.route("/")
async def index(request):
return text("Hello world!")
@app.route("/health")
async def health(request):
if circuit.current_state == "open":
return text("Unavailable", status=503)
return text("OK")