def _get_message(response):
"""
Extracts the message body or a response object by checking for a
json response and returning the reason otherwise getting body.
"""
if _is_json(response.headers.get('content-type', None)):
try:
json = response.json()
return json.get('reason', None)
except (AttributeError, ValueError) as ex:
pass
else:
# if the response is not JSON, return the text content
return response.text
def _get_message(response):
"""
Extracts the message body or a response object by checking for a json response and returning the reason otherwise
getting body.
"""
if _is_json(response.headers.get('content-type', None)):
try:
json = response.json()
return json.get('reason', None)
except (AttributeError, ValueError):
pass
else:
# if the response is not JSON, return the text content
return response.text
def _get_message(response):
"""
Extracts the message body or a response object by checking for a json response and returning the reason otherwise
getting body.
"""
try:
if _is_json(response.headers.get('content-type', None)):
json = response.json()
return json.get('reason', None)
else:
# if the response is not JSON, return the text content
return response.text
except (AttributeError, ValueError):
# The response can be truncated. In which case, the message cannot be retrieved.
return None
def _get_message(response):
"""
Extracts the message body or a response object by checking for a json response and returning the reason otherwise
getting body.
"""
try:
if _is_json(response.headers.get('content-type', None)):
json = response.json()
return json.get('reason', None)
else:
# if the response is not JSON, return the text content
return response.text
except (AttributeError, ValueError):
# The response can be truncated. In which case, the message cannot be retrieved.
return None
def test_is_json():
assert utils._is_json('application/json')
assert utils._is_json('application/json;charset=ISO-8859-1')
assert not utils._is_json('application/flapdoodle;charset=ISO-8859-1')
assert not utils._is_json(None)
assert not utils._is_json('')
def _with_retry(function, verbose=False, \
retry_status_codes=[429, 502, 503, 504], retry_errors=[], retry_exceptions=[], \
retries=3, wait=1, back_off=2, max_wait=30):
"""
Retries the given function under certain conditions.
:param function: A function with no arguments. If arguments are needed, use a lambda (see example).
:param retry_status_codes: What status codes to retry upon in the case of a SynapseHTTPError.
:param retry_errors: What reasons to retry upon, if function().response.json()['reason'] exists.
:param retry_exceptions: What types of exceptions, specified as strings, to retry upon.
:param retries: How many times to retry maximum.
:param wait: How many seconds to wait between retries.
:param back_off: Exponential constant to increase wait for between progressive failures.
:returns: function()
Example::
def foo(a, b, c): return [a, b, c]
result = self._with_retry(lambda: foo("1", "2", "3"), **STANDARD_RETRY_PARAMS)
"""
# Retry until we succeed or run out of tries
while True:
# Start with a clean slate
exc_info = None
retry = False
response = None
# Try making the call
try:
response = function()
except Exception as ex:
exc_info = sys.exc_info()
if verbose:
sys.stderr.write(str(ex.message)+'\n')
if hasattr(ex, 'response'):
response = ex.response
# Check if we got a retry-able error
if response is not None:
if response.status_code in retry_status_codes:
if verbose:
print "retrying on status code: " + str(response.status_code)
retry = True
elif response.status_code not in range(200,299):
## if the response is JSON, look for retryable errors in the 'reason' field
if _is_json(response.headers.get('content-type', None)):
try:
json = response.json()
## special case for message throttling
if 'Please slow down. You may send a maximum of 10 message' in json.get('reason', None):
if verbose: print "retrying", json.get('reason', None)
retry = True
wait = 16
elif any([msg.lower() in json.get('reason', None).lower() for msg in retry_errors]):
if verbose: print "retrying", json.get('reason', None)
retry = True
except (AttributeError, ValueError) as ex:
pass
## if the response is not JSON, look for retryable errors in its text content
elif any([msg.lower() in response.content.lower() for msg in retry_errors]):
if verbose: print "retrying", response.content
retry = True
# Check if we got a retry-able exception
if exc_info is not None:
if exc_info[1].__class__.__name__ in retry_exceptions or any([msg.lower() in str(exc_info[1]).lower() for msg in retry_errors]):
if verbose: print "retrying exception: ", exc_info[1].__class__.__name__, str(exc_info[1])
retry = True
# Wait then retry
retries -= 1
if retries >= 0 and retry:
randomized_wait = wait*random.uniform(0.5,1.5)
if verbose:
sys.stderr.write('\n... Retrying in {wait} seconds...\n'.format(wait=randomized_wait))
time.sleep(randomized_wait)
wait = min(max_wait, wait*back_off)
continue
# Out of retries, re-raise the exception or return the response
if exc_info:
# Re-raise exception, preserving original stack trace
raise exc_info[0], exc_info[1], exc_info[2]
return response
def _with_retry(function, verbose=False, \
retry_status_codes=[429, 502, 503, 504], retry_errors=[], retry_exceptions=[], \
retries=3, wait=1, back_off=2, max_wait=30):
"""
Retries the given function under certain conditions.
:param function: A function with no arguments. If arguments are needed, use a lambda (see example).
:param retry_status_codes: What status codes to retry upon in the case of a SynapseHTTPError.
:param retry_errors: What reasons to retry upon, if function().response.json()['reason'] exists.
:param retry_exceptions: What types of exceptions, specified as strings, to retry upon.
:param retries: How many times to retry maximum.
:param wait: How many seconds to wait between retries.
:param back_off: Exponential constant to increase wait for between progressive failures.
:returns: function()
Example::
def foo(a, b, c): return [a, b, c]
result = self._with_retry(lambda: foo("1", "2", "3"), **STANDARD_RETRY_PARAMS)
"""
# Retry until we succeed or run out of tries
while True:
# Start with a clean slate
exc_info = None
retry = False
response = None
# Try making the call
try:
response = function()
except Exception as ex:
exc_info = sys.exc_info()
if verbose:
sys.stderr.write(str(ex.message) + '\n')
if hasattr(ex, 'response'):
response = ex.response
# Check if we got a retry-able error
if response is not None:
if response.status_code in retry_status_codes:
if verbose:
print "retrying on status code: " + str(
response.status_code)
retry = True
elif response.status_code not in range(200, 299):
## if the response is JSON, look for retryable errors in the 'reason' field
if _is_json(response.headers.get('content-type', None)):
try:
json = response.json()
## special case for message throttling
if 'Please slow down. You may send a maximum of 10 message' in json.get(
'reason', None):
if verbose:
print "retrying", json.get('reason', None)
retry = True
wait = 16
elif any([
msg.lower() in json.get('reason',
None).lower()
for msg in retry_errors
]):
if verbose:
print "retrying", json.get('reason', None)
retry = True
except (AttributeError, ValueError) as ex:
pass
## if the response is not JSON, look for retryable errors in its text content
elif any([
msg.lower() in response.content.lower()
for msg in retry_errors
]):
if verbose: print "retrying", response.content
retry = True
# Check if we got a retry-able exception
if exc_info is not None:
if exc_info[1].__class__.__name__ in retry_exceptions or any([
msg.lower() in str(exc_info[1]).lower()
for msg in retry_errors
]):
if verbose:
print "retrying exception: ", exc_info[
1].__class__.__name__, str(exc_info[1])
retry = True
# Wait then retry
retries -= 1
if retries >= 0 and retry:
randomized_wait = wait * random.uniform(0.5, 1.5)
if verbose:
sys.stderr.write(
'\n... Retrying in {wait} seconds...\n'.format(
wait=randomized_wait))
time.sleep(randomized_wait)
wait = min(max_wait, wait * back_off)
continue
# Out of retries, re-raise the exception or return the response
if exc_info:
# Re-raise exception, preserving original stack trace
raise exc_info[0], exc_info[1], exc_info[2]
return response
def test_is_json():
assert_true(utils._is_json('application/json'))
assert_true(utils._is_json('application/json;charset=ISO-8859-1'))
assert_false(utils._is_json('application/flapdoodle;charset=ISO-8859-1'))
assert_false(utils._is_json(None))
assert_false(utils._is_json(''))
def test_is_json():
assert utils._is_json('application/json')
assert utils._is_json('application/json;charset=ISO-8859-1')
assert not utils._is_json('application/flapdoodle;charset=ISO-8859-1')
assert not utils._is_json(None)
assert not utils._is_json('')
def _raise_for_status(response, verbose=False):
"""
Replacement for requests.response.raise_for_status().
Catches and wraps any Synapse-specific HTTP errors with appropriate text.
"""
message = None
## TODO: Add more meaningful Synapse-specific messages to each error code
## TODO: For some status codes, throw other types of exceptions
if 400 <= response.status_code < 500:
## TODOs:
## 400: 'bad_request'
## 401: 'unauthorized'
## 402: 'payment_required'
## 403: 'forbidden'
## 404: 'not_found'
## 405: 'method_not_allowed'
## 406: 'not_acceptable'
## 407: 'proxy_authentication_required'
## 408: 'request_timeout'
## 409: 'conflict'
## 410: 'gone'
## 411: 'length_required'
## 412: 'precondition_failed'
## 413: 'request_entity_too_large'
## 414: 'request_uri_too_large'
## 415: 'unsupported_media_type'
## 416: 'requested_range_not_satisfiable'
## 417: 'expectation_failed'
## 418: 'im_a_teapot'
## 422: 'unprocessable_entity'
## 423: 'locked'
## 424: 'failed_dependency'
## 425: 'unordered_collection'
## 426: 'upgrade_required'
## 428: 'precondition_required'
## 429: 'too_many_requests'
## 431: 'header_fields_too_large'
## 444: 'no_response'
## 449: 'retry_with'
## 450: 'blocked_by_windows_parental_controls'
## 451: 'unavailable_for_legal_reasons'
## 499: 'client_closed_request'
message = '%s Client Error: %s' % (response.status_code, response.reason)
elif 500 <= response.status_code < 600:
## TODOS:
## 500: 'internal_server_error'
## 501: 'not_implemented'
## 502: 'bad_gateway'
## 503: 'service_unavailable'
## 504: 'gateway_timeout'
## 505: 'http_version_not_supported'
## 506: 'variant_also_negotiates'
## 507: 'insufficient_storage'
## 509: 'bandwidth_limit_exceeded'
## 510: 'not_extended'
message = '%s Server Error: %s' % (response.status_code, response.reason)
if message is not None:
if utils._is_json(response.headers.get('content-type',None)):
# Append the server's JSON error message
message += "\n%s" % response.json()['reason']
if verbose:
try:
# Append the request sent
message += "\n\n>>>>>> Request <<<<<<\n%s %s" % (response.request.url, response.request.method)
message += "\n>>> Headers: %s" % response.request.headers
message += "\n>>> Body: %s" % response.request.body
except: message += "\nCould not append all request info"
try:
# Append the response recieved
message += "\n\n>>>>>> Response <<<<<<\n%s" % str(response)
message += "\n>>> Headers: %s" % response.headers
message += "\n>>> Body: %s\n\n" % response.text
except: message += "\nCould not append all response info"
raise SynapseHTTPError(message, response=response)
def _with_retry(function, verbose=False, \
retry_status_codes=[502,503], retry_errors=[], retry_exceptions=[], \
retries=3, wait=1, back_off=2):
"""
Retries the given function under certain conditions.
:param function: A function with no arguments. If arguments are needed, use a lambda (see example).
:param retry_status_codes: What status codes to retry upon in the case of a SynapseHTTPError.
:param retry_errors: What reasons to retry upon, if function().response.json()['reason'] exists.
:param retry_exceptions: What types of exceptions, specified as strings, to retry upon.
:param retries: How many times to retry maximum.
:param wait: How many seconds to wait between retries.
:param back_off: Exponential constant to increase wait for between progressive failures.
:returns: function()
Example::
def foo(a, b, c): return [a, b, c]
result = self._with_retry(lambda: foo("1", "2", "3"), **STANDARD_RETRY_PARAMS)
"""
# Retry until we succeed or run out of tries
while True:
# Start with a clean slate
exc_info = None
retry = False
response = None
# Try making the call
try:
response = function()
except Exception as ex:
exc_info = sys.exc_info()
if verbose:
sys.stderr.write(
ex.message +
'\n') # This message will contain lots of info
if hasattr(ex, 'response'):
response = ex.response
# Check if we got a retry-able error
if response is not None:
if response.status_code not in range(200, 299):
if response.status_code in retry_status_codes:
retry = True
elif _is_json(response.headers.get('content-type', None)):
try:
json = response.json()
if json.get('reason', None) in retry_errors:
retry = True
except (AttributeError, ValueError) as ex:
pass
elif any([msg in response.content for msg in retry_errors]):
retry = True
# Check if we got a retry-able exception
if exc_info is not None and exc_info[
1].__class__.__name__ in retry_exceptions:
retry = True
# Wait then retry
retries -= 1
if retries >= 0 and retry:
if verbose:
sys.stderr.write('\n... Retrying in %d seconds...\n' % wait)
time.sleep(wait)
wait *= back_off
continue
# Out of retries, re-raise the exception or return the response
if exc_info:
# Re-raise exception, preserving original stack trace
raise exc_info[0], exc_info[1], exc_info[2]
return response
def _raise_for_status(response, verbose=False):
"""
Replacement for requests.response.raise_for_status().
Catches and wraps any Synapse-specific HTTP errors with appropriate text.
"""
message = None
## TODO: Add more meaningful Synapse-specific messages to each error code
## TODO: For some status codes, throw other types of exceptions
if 400 <= response.status_code < 500:
## TODOs:
## 400: 'bad_request'
## 401: 'unauthorized'
## 402: 'payment_required'
## 403: 'forbidden'
## 404: 'not_found'
## 405: 'method_not_allowed'
## 406: 'not_acceptable'
## 407: 'proxy_authentication_required'
## 408: 'request_timeout'
## 409: 'conflict'
## 410: 'gone'
## 411: 'length_required'
## 412: 'precondition_failed'
## 413: 'request_entity_too_large'
## 414: 'request_uri_too_large'
## 415: 'unsupported_media_type'
## 416: 'requested_range_not_satisfiable'
## 417: 'expectation_failed'
## 418: 'im_a_teapot'
## 422: 'unprocessable_entity'
## 423: 'locked'
## 424: 'failed_dependency'
## 425: 'unordered_collection'
## 426: 'upgrade_required'
## 428: 'precondition_required'
## 429: 'too_many_requests'
## 431: 'header_fields_too_large'
## 444: 'no_response'
## 449: 'retry_with'
## 450: 'blocked_by_windows_parental_controls'
## 451: 'unavailable_for_legal_reasons'
## 499: 'client_closed_request'
message = '%s Client Error: %s' % (response.status_code, response.reason)
elif 500 <= response.status_code < 600:
## TODOS:
## 500: 'internal_server_error'
## 501: 'not_implemented'
## 502: 'bad_gateway'
## 503: 'service_unavailable'
## 504: 'gateway_timeout'
## 505: 'http_version_not_supported'
## 506: 'variant_also_negotiates'
## 507: 'insufficient_storage'
## 509: 'bandwidth_limit_exceeded'
## 510: 'not_extended'
message = '%s Server Error: %s' % (response.status_code, response.reason)
if message is not None:
if utils._is_json(response.headers.get('content-type',None)):
# Append the server's JSON error message
message += "\n%s" % response.json()['reason']
if verbose:
try:
# Append the request sent
message += "\n\n>>>>>> Request <<<<<<\n%s %s" % (response.request.url, response.request.method)
message += "\n>>> Headers: %s" % response.request.headers
message += "\n>>> Body: %s" % response.request.body
except: message += "\nCould not append all request info"
try:
# Append the response recieved
message += "\n\n>>>>>> Response <<<<<<\n%s" % str(response)
message += "\n>>> Headers: %s" % response.headers
message += "\n>>> Body: %s\n\n" % response.text
except: message += "\nCould not append all response info"
raise SynapseHTTPError(message, response=response)
def _with_retry(function, verbose=False, \
retry_status_codes=[502,503], retry_errors=[], retry_exceptions=[], \
retries=3, wait=1, back_off=2):
"""
Retries the given function under certain conditions.
:param function: A function with no arguments. If arguments are needed, use a lambda (see example).
:param retry_status_codes: What status codes to retry upon in the case of a SynapseHTTPError.
:param retry_errors: What reasons to retry upon, if function().response.json()['reason'] exists.
:param retry_exceptions: What types of exceptions, specified as strings, to retry upon.
:param retries: How many times to retry maximum.
:param wait: How many seconds to wait between retries.
:param back_off: Exponential constant to increase wait for between progressive failures.
:returns: function()
Example::
def foo(a, b, c): return [a, b, c]
result = self._with_retry(lambda: foo("1", "2", "3"), **STANDARD_RETRY_PARAMS)
"""
# Retry until we succeed or run out of tries
while True:
# Start with a clean slate
exc_info = None
retry = False
response = None
# Try making the call
try:
response = function()
except Exception as ex:
exc_info = sys.exc_info()
if verbose:
sys.stderr.write(ex.message+'\n') # This message will contain lots of info
if hasattr(ex, 'response'):
response = ex.response
# Check if we got a retry-able error
if response is not None:
if response.status_code not in range(200,299):
if response.status_code in retry_status_codes:
retry = True
elif _is_json(response.headers.get('content-type', None)):
try:
json = response.json()
if json.get('reason', None) in retry_errors:
retry = True
except (AttributeError, ValueError) as ex:
pass
elif any([msg in response.content for msg in retry_errors]):
retry = True
# Check if we got a retry-able exception
if exc_info is not None and exc_info[1].__class__.__name__ in retry_exceptions:
retry = True
# Wait then retry
retries -= 1
if retries >= 0 and retry:
if verbose:
sys.stderr.write('\n... Retrying in %d seconds...\n' % wait)
time.sleep(wait)
wait *= back_off
continue
# Out of retries, re-raise the exception or return the response
if exc_info:
# Re-raise exception, preserving original stack trace
raise exc_info[0], exc_info[1], exc_info[2]
return response
def test_is_json():
assert_true(utils._is_json('application/json'))
assert_true(utils._is_json('application/json;charset=ISO-8859-1'))
assert_false(utils._is_json('application/flapdoodle;charset=ISO-8859-1'))
assert_false(utils._is_json(None))
assert_false(utils._is_json(''))
