def __init__(self):
retry_policy = RetryPolicy(allowed_retries=4)
self.failsafe = Failsafe(retry_policy=retry_policy,
circuit_breaker=CircuitBreaker(maximum_failures=8))
self.failsafe_fallback = Failsafe(retry_policy=retry_policy,
circuit_breaker=CircuitBreaker(maximum_failures=8))
def test_circuit_breaker_opened_circuit_has_no_cause(self):
failing_operation = create_failing_operation()
try:
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
circuit_breaker.open()
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
raise Exception("Expected CircuitOpen exception")
except CircuitOpen as e:
assert e.__cause__ is None
assert failing_operation.called == 0
def test_circuit_breaker_with_abort(self):
aborting_operation = create_aborting_operation()
policy = RetryPolicy(abortable_exceptions=[SomeAbortableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
circuit_breaker.record_failure = MagicMock()
with pytest.raises(SomeAbortableException):
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(aborting_operation))
circuit_breaker.record_failure.assert_not_called()
assert aborting_operation.called == 1
def __init__(self,
url,
auth=None,
session=None,
trailing_slash=False,
retries=6,
autopaginate=None):
if auth is not None and not isinstance(auth, aiohttp.BasicAuth):
auth = aiohttp.BasicAuth(*auth)
super(GenericClient, self).__init__(url, auth, session, trailing_slash,
autopaginate)
max_failures = retries - 1
circuit_breaker = CircuitBreaker(maximum_failures=max_failures)
retry_policy = RetryPolicy(
allowed_retries=retries,
retriable_exceptions=[ClientConnectionError],
abortable_exceptions=[
exceptions.BadRequestError,
exceptions.NotAuthenticatedError,
exceptions.ResourceNotFound,
exceptions.MultipleResourcesFound,
exceptions.HTTPError,
ValueError,
],
)
self.failsafe = Failsafe(circuit_breaker=circuit_breaker,
retry_policy=retry_policy)
def test_circuit_breaker_with_retries(self):
failing_operation = create_failing_operation()
with pytest.raises(CircuitOpen):
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
assert failing_operation.called == 2
def test_circuit_breaker_circuitopener_raises_circuitopen_with_cause(self):
original_exception = SomeRetriableException("My Error Message")
failing_operation = create_failing_operation(original_exception)
try:
policy = RetryPolicy(5, [SomeRetriableException])
circuit_breaker = CircuitBreaker(maximum_failures=2)
loop.run_until_complete(
Failsafe(
retry_policy=policy,
circuit_breaker=circuit_breaker).run(failing_operation))
raise Exception("Expected CircuitOpen exception")
except CircuitOpen as e:
assert e.__cause__ is original_exception
def __init__(self, fallback_options, retry_policy_factory=None, circuit_breaker_factory=None):
"""
:param fallback_options: a list of objects which will differentiate between different fallback calls. An item
from this list will be passed as the first parameter to the function provided to the run method.
:param retry_policy_factory: factory function accepting a fallback option
and returning a retry policy
:param circuit_breaker_factory: factory function accepting a fallback option
and returning a circuit breaker
"""
retry_policy_factory = retry_policy_factory or (lambda _: RetryPolicy())
circuit_breaker_factory = circuit_breaker_factory or (lambda _: CircuitBreaker())
def _create_failsafe(option):
return Failsafe(retry_policy=retry_policy_factory(option),
circuit_breaker=circuit_breaker_factory(option))
self.failsafes = [(option, _create_failsafe(option))
for option in fallback_options]
def __init__(self):
self.failsafe = Failsafe(
retry_policy=RetryPolicy(allowed_retries=4,
abortable_exceptions=[NotFoundError]),
circuit_breaker=CircuitBreaker(maximum_failures=8))
from .utils import apply_filters, remove_script_tags, remove_meta_fragment_tag, is_yesish
logger = logging.getLogger(__name__)
executor = ThreadPoolExecutor(max_workers=cpu_count() * 5)
HTML_FILTERS: Tuple[Callable[[str], str]] = (remove_script_tags,
remove_meta_fragment_tag)
ALLOWED_DOMAINS: Set = set(
dm.strip() for dm in os.getenv('ALLOWED_DOMAINS', '').split(',')
if dm.strip())
CACHE_LIVE_TIME: int = int(os.getenv('CACHE_LIVE_TIME', 3600))
SENTRY_DSN: Optional[str] = os.getenv('SENTRY_DSN')
_ENABLE_CB = is_yesish(os.getenv('ENABLE_CIRCUIT_BREAKER', '0'))
_CB_FAIL_MAX: int = int(os.getenv('CIRCUIT_BREAKER_FAIL_MAX', 5))
_CB_RESET_TIMEOUT: int = int(os.getenv('CIRCUIT_BREAKER_RESET_TIMEOUT', 60))
_BREAKERS = defaultdict(lambda: Failsafe(circuit_breaker=CircuitBreaker(
maximum_failures=_CB_FAIL_MAX, reset_timeout_seconds=_CB_RESET_TIMEOUT)))
if SENTRY_DSN:
sentry = raven.Client(
SENTRY_DSN,
transport=AioHttpTransport,
release=raven.fetch_package_version('prerender'),
site='Prerender',
)
else:
sentry = None
def _save_to_cache(key: str, data: bytes, format: str = 'html') -> None:
try:
cache.set(key, data, CACHE_LIVE_TIME, format)
# -*- coding: utf-8 -*-
import os
import aiohttp
from failsafe import CircuitBreaker, Failsafe, FailsafeError, RetryPolicy
from sanic import Sanic
from sanic.response import json, text
from sanic_prometheus import monitor
app = Sanic()
circuit = CircuitBreaker(
maximum_failures=3,
reset_timeout_seconds=10
)
failsafe = Failsafe(
circuit_breaker=circuit,
retry_policy=RetryPolicy()
)
@app.route("/")
async def index(request):
return text("Hello world!")
@app.route("/health")
async def health(request):
if circuit.current_state == "open":
return text("Unavailable", status=503)