class TokenHandler(IPythonHandler):
@retry(wait=wait_fixed(3),
retry=(retry_if_exception_type(RuntimeError)
& retry_if_exception_message(
match=r'[Nn]o authorization code found.*')),
stop=stop_after_delay(AssembleApp.instance().access_token_timeout))
async def get_token(self, url, data):
response = requests.post(url, json=data)
d = response.json()
if response.ok:
# TODO also store other token info
return d['access_token']
else:
reason = d.get('message', '').lower()
raise RuntimeError(reason)
@tornado.web.authenticated
async def post(self):
"""request token if we have just authenticated"""
app = AssembleApp.instance()
base = app.oauth_gateway_url
state = app.state
url = urljoin(base, '/api/v1/access_token')
data = {'state': state}
try:
token = await self.get_token(url, data)
app.set_github_token(token)
self.finish()
except RetryError:
self.send_error(status_code=400, reason='timeout')
except RuntimeError as e:
self.send_error(status_code=400, reason=str(e))
app.reset_state()
def retry_on_db_error(func, retry=None):
"""Decorates the given function so that it retries on DB errors.
Note that the decorator retries the function/method only on some
of the DB errors that are considered to be worth retrying, like
deadlocks and disconnections.
:param func: Function to decorate.
:param retry: a Retrying object
:return: Decorated function.
"""
if not retry:
retry = tenacity.Retrying(
retry=(
tenacity.retry_if_exception_type(_RETRY_ERRORS) |
tenacity.retry_if_exception_message(
match='Too many connections'
)
),
stop=tenacity.stop_after_attempt(50),
wait=tenacity.wait_incrementing(start=0, increment=0.1, max=2)
)
# The `assigned` arg should be empty as some of the default values are not
# supported by simply initialized MagicMocks. The consequence may
# be that the representation will contain the wrapper and not the
# wrapped function.
@functools.wraps(func, assigned=[])
def decorate(*args, **kw):
# Retrying library decorator might potentially run a decorated
# function within a new thread so it's safer not to apply the
# decorator directly to a target method/function because we can
# lose an authentication context.
# The solution is to create one more function and explicitly set
# auth context before calling it (potentially in a new thread).
auth_ctx = context.ctx() if context.has_ctx() else None
return retry.call(_with_auth_context, auth_ctx, func, *args, **kw)
return decorate
def test_retry_if_exception_message_negative_too_many_inputs(self):
with self.assertRaises(TypeError):
tenacity.retry_if_exception_message(message="negative",
match="negative")
def test_retry_if_exception_message_negative_no_inputs(self):
with self.assertRaises(TypeError):
tenacity.retry_if_exception_message()
return thing.go()
@retry(stop=tenacity.stop_after_attempt(3),
retry=tenacity.retry_unless_exception_type(NameError))
def _retryable_test_with_unless_exception_type_name_attempt_limit(thing):
return thing.go()
@retry(retry=tenacity.retry_unless_exception_type())
def _retryable_test_with_unless_exception_type_no_input(thing):
return thing.go()
@retry(stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_message(
message=NoCustomErrorAfterCount.derived_message))
def _retryable_test_if_exception_message_message(thing):
return thing.go()
@retry(retry=tenacity.retry_if_not_exception_message(
message=NoCustomErrorAfterCount.derived_message))
def _retryable_test_if_not_exception_message_message(thing):
return thing.go()
@retry(retry=tenacity.retry_if_exception_message(
match=NoCustomErrorAfterCount.derived_message[:3] + ".*"))
def _retryable_test_if_exception_message_match(thing):
return thing.go()
class RetryRequest:
"""
Make requests to a URL, but retry under certain conditions to tolerate server graceful shutdown.
"""
def __init__(self, request, method, url, auth, request_headers,
request_json, return_json):
self.request = request
self.method = method
self.url = url
self.auth = auth
self.headers = request_headers
self.json = request_json
self.return_json = return_json
def __handle_response(self, response):
try:
response.raise_for_status()
except ClientResponseError as ex:
raise ex
else:
logger.debug('successfully connected to service',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'],
url=self.url)
@retry(reraise=True,
stop=stop_after_attempt(basic_attempt_limit),
wait=wait_exponential(multiplier=wait_multiplier, exp_base=25),
after=after_failed_basic,
retry=(retry_if_exception_message(match='503.*')
| retry_if_exception_type(
(ClientConnectionError, ClientConnectorError))))
async def _request_basic(self):
# basic request without keep-alive to avoid terminating service.
logger.info('request using basic connection',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'])
async with aiohttp.request(self.method,
self.url,
auth=self.auth,
json=self.json,
headers=self.headers) as resp:
self.__handle_response(resp)
if self.return_json:
return await resp.json()
else:
return None
@retry(stop=stop_after_attempt(pooled_attempts_limit),
wait=wait_exponential(multiplier=wait_multiplier),
after=after_failed_pooled,
retry=(retry_if_exception_message(match='503.*')
| retry_if_exception_type(
(ClientConnectionError, ClientConnectorError))))
async def _request_using_pool(self):
async with self.request.app.http_session_pool.request(
self.method,
self.url,
auth=self.auth,
json=self.json,
headers=self.headers,
ssl=False) as resp:
self.__handle_response(resp)
if self.return_json:
return await resp.json()
else:
return None
async def make_request(self):
"""
Make a request with retries.
First the fast pooled connection will be tried, but if certain failures are detected, then it will be retried.
If the retry limit is reached then a basic connection will be tried (and retried if necessary)
Finally the error will be propagated.
"""
logger.debug('making request with handler',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'],
method=self.method,
url=self.url)
try:
try:
return await self._request_using_pool()
except RetryError as retry_ex:
attempts = retry_ex.last_attempt.attempt_number
logger.warn(
'Could not make request using normal pooled connection',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'],
attempts=attempts)
return await self._request_basic()
except ClientResponseError as ex:
if ex.status not in [400, 404, 429]:
logger.error('error in response',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'],
url=self.url,
status_code=ex.status)
elif ex.status == 429:
self.log_too_many_requests(ex)
elif ex.status == 400:
logger.warn('bad request',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'],
url=self.url,
status_code=ex.status)
raise ex
except (ClientConnectionError, ClientConnectorError) as ex:
logger.error('client failed to connect',
client_ip=self.request['client_ip'],
client_id=self.request['client_id'],
trace=self.request['trace'],
url=self.url)
raise ex
def log_too_many_requests(self, ex: ClientResponseError):
ai_svc_url = self.request.app['ADDRESS_INDEX_SVC_URL']
tracking = {
"client_ip": self.request['client_ip'],
"client_id": self.request['client_id'],
"trace": self.request['trace']
}
if ai_svc_url in self.url:
logger.error('error in AIMS response',
**tracking,
url=self.url,
status_code=ex.status)
else:
logger.warn('too many requests',
**tracking,
url=self.url,
status_code=ex.status)
else:
thank_you = ""
context = {
"order": order,
"thank_you": thank_you,
"item_type": item_type,
"prep": prep
}
if seller:
context["seller"] = True
return render_to_string("shop/email.html", context=context)
@retry(
wait=wait_exponential(multiplier=1, min=4, max=10),
stop=stop_after_attempt(4),
retry=retry_if_exception_message(message='RETRY')
)
def send_email(recipient, subject, text):
response = requests.post(
settings.MAILGUN_BASE_URL + "messages",
auth=("api", settings.MAILGUN_PRIVATE_KEY),
data={
"from": settings.MAILGUN_SENDER,
"to": [recipient],
"subject": subject,
"html": text
}
)
if response.status_code == 200:
return {'status': response.status_code}
elif response.status_code == 400:
# third parties libraries
import tenacity
from socket import timeout
# whre all the bad requests are tried again...
# https://developers.google.com/drive/api/v3/handle-errors#exponential-backoff
MAX_ATTEMPTS = 30
EXP_MULTIPLIER = 0.5
EXP_MAX_WAIT = 60
# In what case should tenacity try again?
retry_exceptions = (
# https://developers.google.com/drive/api/v3/handle-errors#403_user_rate_limit_exceeded
tenacity.retry_if_exception_message(match=r".+?User Rate Limit Exceeded\.")
| tenacity.retry_if_exception_message(match=r".+?Rate limit exceeded\.")
# https://developers.google.com/drive/api/v3/handle-errors#500_backend_error
| tenacity.retry_if_exception_message(match=r".+?Internal Error")
| tenacity.retry_if_exception_message(match=r".+?Transient failure")
| tenacity.retry_if_exception_message(
match=r".+?The read operation timed out")
| tenacity.retry_if_exception_message(match=r".+?HttpError 500")
| tenacity.retry_if_exception_type(timeout)
| tenacity.retry_if_exception_type(ConnectionResetError))
def before_sleep_print(retry_state):
"""Before call strategy that logs to some logger the attempt."""
# third parties libraries
import tenacity
# whre all the bad requests are tried again...
# https://developers.google.com/drive/api/v3/handle-errors#exponential-backoff
MAX_ATTEMPTS = 30
EXP_MULTIPLIER = 0.5
EXP_MAX_WAIT = 60
# In what case should tenacity try again?
retry_exceptions = (
# https://developers.google.com/drive/api/v3/handle-errors#403_user_rate_limit_exceeded
tenacity.retry_if_exception_message(match=r".+?User Rate Limit Exceeded\.")
|
# https://developers.google.com/drive/api/v3/handle-errors#500_backend_error
tenacity.retry_if_exception_message(match=r".+?Internal Error"))
@tenacity.retry(stop=tenacity.stop_after_attempt(MAX_ATTEMPTS),
wait=tenacity.wait_exponential(multiplier=EXP_MULTIPLIER,
max=EXP_MAX_WAIT),
retry=retry_exceptions)
def call_endpoint(endpoint, params):
return endpoint(**params).execute()
@tenacity.retry(stop=tenacity.stop_after_attempt(MAX_ATTEMPTS),
wait=tenacity.wait_exponential(multiplier=EXP_MULTIPLIER,
max=EXP_MAX_WAIT),
retry=retry_exceptions)
def execute_request(request):
