def _request_with_retry(self, verb, url, headers, data, max_tries = 5):
durations = _exponential_backoff(max_tries)
for index, length in enumerate(durations):
if length > 0:
log.debug('Sleeping for %f seconds before retrying', length)
time.sleep(length)
try:
if verb == HTTP_GET:
response = requests.get(url, headers = headers, verify=True)
elif verb == HTTP_POST:
response = requests.post(url, data = data, headers = headers, verify=True)
else:
raise AttributeError('Not supported HTTP action "{0}".'.format(verb))
# check status code
if response.status_code != 200:
http_code = response.status_code
try:
aws_code = response.json['Error']['Code']
message = response.json['Error']['Message']
except TypeError as ex:
raise AttributeError('HTTP {0}: {1}'.format(http_code, response.text))
if http_code < 500 and aws_code != AwsErrorCode.Throttling:
raise AwsServiceException(message, aws_code, http_code)
elif AwsErrorCode.Throttling == aws_code or http_code == 503:
prompt.info('Request is throttled.')
_extend_backoff(durations)
if index + 1 < len(durations):
prompt.info('Error {0}:{1}. Wait {2} seconds before retry.'.\
format(aws_code, message, durations[index + 1]))
log.error(message + ' ' + aws_code)
last_message = message
else:
if response.json is None:
raise ValueError('Cannot parse response form {0}.'.format(response.url))
return response
except requests.exceptions.SSLError as ex:
log.error('SSL Error: %s', ex)
raise
except (requests.exceptions.HTTPError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as ex:
log.error(ex)
last_message = ex
else:
if aws_code is None:
aws_code = ''
if http_code is None:
http_code = ''
raise AwsServiceException(last_message, aws_code, http_code)
def _urlopen_withretry(self,
request_or_url,
max_tries=5,
http_error_extractor=_extractAwsErrorMessage,
opener=_default_opener):
"""
Exponentially retries up to max_tries to open request_or_url.
Raises an IOError on failure
http_error_extractor is a function that takes a urllib2.HTTPError and returns a tuple of
(AWS error code, message)
"""
durations = _exponential_backoff(max_tries)
for index, length in enumerate(durations):
if length > 0:
log.debug(u'Sleeping for %f seconds before retrying', length)
time.sleep(length)
try:
return opener.open(request_or_url)
except urllib2.HTTPError as ex:
http_code = ex.code
aws_code, message = http_error_extractor(ex)
if http_code < 500 and aws_code != AwsErrorCode.Throttling:
raise AwsServiceException(message, aws_code, http_code)
elif AwsErrorCode.Throttling == aws_code or http_code == 503:
_extend_backoff(durations)
if index + 1 < len(durations):
prompt.info(u'Error {0}:{1}. Wait {2} seconds before retry.'.\
format(aws_code, message, durations[index + 1]))
log.error(message + u' ' + aws_code)
last_message = message
except urllib2.URLError as url_ex:
log.error(u'URLError: %s', url_ex.reason)
raise
else:
raise AwsServiceException(last_message, aws_code, http_code)