def test_raising_IRetryableError_type_is_caught(config):
from pyramid_retry import mark_error_retryable
class MyRetryableError(Exception):
pass
mark_error_retryable(MyRetryableError)
calls = []
def final_view(request):
calls.append('ok')
return 'ok'
def bad_view(request):
calls.append('fail')
raise MyRetryableError
config.add_view(bad_view, last_retry_attempt=False)
config.add_view(final_view, last_retry_attempt=True, renderer='string')
app = config.make_wsgi_app()
app = webtest.TestApp(app)
response = app.get('/')
assert response.body == b'ok'
assert calls == ['fail', 'fail', 'ok']
# contains a unique, user provided value, you can get into a race condition where both
# requests check the database, see nothing with that value exists, then both attempt to
# insert it. One of the requests will succeed, the other will fail with an
# IntegrityError. Retrying the request that failed will then have it see the object
# created by the other request, and will have it do the appropriate action in that case.
#
# The most common way to run into this, is when submitting a form in the browser, if the
# user clicks twice in rapid succession, the browser will send two almost identical
# requests at basically the same time.
#
# One possible issue that this raises, is that it will slow down "legitimate"
# IntegrityError because they'll have to fail multiple times before they ultimately
# fail. We consider this an acceptable trade off, because deterministic IntegrityError
# should be caught with proper validation prior to submitting records to the database
# anyways.
pyramid_retry.mark_error_retryable(IntegrityError)
# A generic wrapper exception that we'll raise when the database isn't available, we
# use this so we can catch it later and turn it into a generic 5xx error.
class DatabaseNotAvailableError(Exception):
...
# We'll add a basic predicate that won't do anything except allow marking a
# route as read only (or not).
class ReadOnlyPredicate:
def __init__(self, val, config):
self.val = val
def text(self):
from pyramid.util import DottedNameResolver
import transaction
import warnings
import zope.interface
try:
from pyramid_retry import IRetryableError
except ImportError: # pragma: no cover
IRetryableError = zope.interface.Interface
try:
from pyramid_retry import mark_error_retryable
except ImportError: # pragma: no cover
mark_error_retryable = lambda error: None
mark_error_retryable(transaction.interfaces.TransientError)
from .compat import reraise
from .compat import text_
resolver = DottedNameResolver(None)
def default_commit_veto(request, response):
"""
When used as a commit veto, the logic in this function will cause the
transaction to be aborted if:
- An ``X-Tm`` response header with the value ``abort`` (or any value
other than ``commit``) exists.
from pyramid_retry import mark_error_retryable
from sqlalchemy.dialects import postgresql as pg
from sqlalchemy.ext.hybrid import hybrid_property
from h.db import Base, mixins
from h.models.annotation import Annotation
from h.util.uri import normalize as uri_normalize
log = logging.getLogger(__name__)
class ConcurrentUpdateError(Exception):
"""Raised when concurrent updates to document data conflict."""
mark_error_retryable(ConcurrentUpdateError)
class Document(Base, mixins.Timestamps):
__tablename__ = "document"
id = sa.Column(sa.Integer, autoincrement=True, primary_key=True)
#: The denormalized value of the first DocumentMeta record with type title.
title = sa.Column("title", sa.UnicodeText())
#: The denormalized value of the "best" http(s) DocumentURI for this Document.
web_uri = sa.Column("web_uri", sa.UnicodeText())
# FIXME: This relationship should be named `uris` again after the
# dependency on the annotator-store is removed, as it clashes with
ContenuArticleModifie, PresentationArticleModifiee, TitreArticleModifie,
)
from .events.base import Event # noqa
from .events.lecture import ( # noqa
AmendementsAJour, AmendementsNonRecuperes, AmendementsNonTrouves,
AmendementsRecuperes, AmendementsRecuperesLiasse, ArticlesRecuperes,
LectureCreee, ReponsesImportees, ReponsesImporteesJSON, SharedTableCreee,
SharedTableRenommee, SharedTableSupprimee,
)
from .lecture import Lecture # noqa
from .phase import Phase # noqa
from .table import SharedTable, UserTable # noqa
from .texte import Texte, TypeTexte # noqa
from .users import AllowedEmailPattern, Team, User # noqa
mark_error_retryable(IntegrityError)
def _get_one(model: Any,
options: Any = None,
**kwargs: Any) -> Tuple[Any, bool]:
query = DBSession.query(model).filter_by(**kwargs)
if options is not None:
query = query.options(options)
return query.one(), False
def _create(model: Any,
create_kwargs: Any = None,
**kwargs: Any) -> Tuple[Any, bool]:
kwargs.update(create_kwargs or {})
def test_mark_error_retryable_on_non_error():
from pyramid_retry import mark_error_retryable
with pytest.raises(ValueError):
mark_error_retryable('some string')
def bad_view(request):
calls.append('fail')
ex = Exception()
mark_error_retryable(ex)
raise ex
def create_app(global_config, **settings): # pylint: disable=unused-argument
config = configure(settings=settings)
# Make sure that pyramid_exclog's tween runs under pyramid_tm's tween so
# that pyramid_exclog doesn't re-open the DB session after pyramid_tm has
# already closed it.
config.add_tween(
"pyramid_exclog.exclog_tween_factory", under="pyramid_tm.tm_tween_factory"
)
config.add_settings({"exclog.extra_info": True})
config.include("pyramid_exclog")
config.include("pyramid_jinja2")
config.include("pyramid_services")
# Use pyramid_tm's explicit transaction manager.
#
# This means that trying to access a request's transaction after pyramid_tm
# has closed the request's transaction will crash, rather than implicitly
# opening a new transaction that doesn't get closed (and potentially
# leaking open DB connections).
#
# This is recommended in the pyramid_tm docs:
#
# https://docs.pylonsproject.org/projects/pyramid_tm/en/latest/#custom-transaction-managers
config.registry.settings["tm.manager_hook"] = pyramid_tm.explicit_manager
config.include("pyramid_tm")
config.include("pyramid_retry")
# Mark all sqlalchemy IntegrityError's as retryable.
#
# This means that if any request fails with any IntegrityError error then
# pyramid_retry will re-try the request up to two times. No error response
# will be sent back to the client, and no crash reported to Sentry, unless
# the request fails three times in a row (or one of the re-tries fails with
# a non-retryable error).
#
# This does mean that if a request is failing with a non-transient
# IntegrityError (so the request has no hope of succeeding on retry) then
# we will pointlessly retry the request twice before failing.
#
# But we shouldn't have too many non-transient IntegrityError's anyway
# (sounds like a bug) and marking all IntegrityError's as retryable means
# that in all cases when an IntegrityError *is* transient and the request
# *can* succeed on retry, it will be retried, without having to mark those
# IntegrityErrors as retryable on a case-by-case basis.
#
# Examples of transient/retryable IntegrityError's are when doing either
# upsert or create-if-not-exists logic when entering rows into the DB:
# concurrent requests can both see that the DB row doesn't exist yet and
# try to create the DB row at the same time and one of them will fail. If
# retried the failed request will now see that the DB row already exists
# and not try to create it, and the request will succeed.
pyramid_retry.mark_error_retryable(IntegrityError)
config.include("lms.authentication")
config.include("lms.extensions.feature_flags")
config.add_feature_flag_providers(
"lms.extensions.feature_flags.config_file_provider",
"lms.extensions.feature_flags.envvar_provider",
"lms.extensions.feature_flags.cookie_provider",
"lms.extensions.feature_flags.query_string_provider",
)
config.include("lms.sentry")
config.include("lms.session")
config.include("lms.models")
config.include("lms.db")
config.include("lms.routes")
config.include("lms.assets")
config.include("lms.views")
config.include("lms.services")
config.include("lms.validation")
config.include("lms.tweens")
config.add_static_view(name="export", path="lms:static/export")
config.add_static_view(name="static", path="lms:static")
config.registry.settings["jinja2.filters"] = {
"static_path": "pyramid_jinja2.filters:static_path_filter",
"static_url": "pyramid_jinja2.filters:static_url_filter",
}
config.action(None, configure_jinja2_assets, args=(config,))
config.scan()
return config.make_wsgi_app()
# contains a unique, user provided value, you can get into a race condition where both
# requests check the database, see nothing with that value exists, then both attempt to
# insert it. One of the requests will succeed, the other will fail with an
# IntegrityError. Retrying the request that failed will then have it see the object
# created by the other request, and will have it do the appropriate action in that case.
#
# The most common way to run into this, is when submitting a form in the browser, if the
# user clicks twice in rapid succession, the browser will send two almost identical
# requests at basically the same time.
#
# One possible issue that this raises, is that it will slow down "legitimate"
# IntegrityError because they'll have to fail multiple times before they ultimately
# fail. We consider this an acceptable trade off, because deterministic IntegrityError
# should be caught with proper validation prior to submitting records to the database
# anyways.
pyramid_retry.mark_error_retryable(IntegrityError)
# A generic wrapper exception that we'll raise when the database isn't available, we
# use this so we can catch it later and turn it into a generic 5xx error.
class DatabaseNotAvailable(Exception):
...
# We'll add a basic predicate that won't do anything except allow marking a
# route as read only (or not).
class ReadOnlyPredicate:
def __init__(self, val, config):
self.val = val
def text(self):