def universal_retry(method, ResourceARNList, **kw):
"""Retry support for resourcegroup tagging apis.
The resource group tagging api typically returns a 200 status code
with embedded resource specific errors. To enable resource specific
retry on throttles, we extract those, perform backoff w/ jitter and
continue. Other errors are immediately raised.
We do not aggregate unified resource responses across retries, only the
last successful response is returned for a subset of the resources if
a retry is performed.
"""
max_attempts = 6
for idx, delay in enumerate(utils.backoff_delays(1.5, 2**8, jitter=True)):
response = method(ResourceARNList=ResourceARNList, **kw)
failures = response.get('FailedResourcesMap', {})
if not failures:
return response
errors = {}
throttles = set()
for f_arn in failures:
error_code = failures[f_arn]['ErrorCode']
if error_code == 'ThrottlingException':
throttles.add(f_arn)
elif error_code == 'ResourceNotFoundException':
continue
else:
errors[f_arn] = error_code
if errors:
raise Exception("Resource Tag Errors %s" % (errors))
if idx == max_attempts - 1:
raise Exception("Resource Tag Throttled %s" %
(", ".join(throttles)))
time.sleep(delay)
ResourceARNList = list(throttles)
def universal_retry(method, ResourceARNList, **kw):
"""Retry support for resourcegroup tagging apis.
The resource group tagging api typically returns a 200 status code
with embedded resource specific errors. To enable resource specific
retry on throttles, we extract those, perform backoff w/ jitter and
continue. Other errors are immediately raised.
We do not aggregate unified resource responses across retries, only the
last successful response is returned for a subset of the resources if
a retry is performed.
"""
max_attempts = 6
for idx, delay in enumerate(
utils.backoff_delays(1.5, 2 ** 8, jitter=True)):
response = method(ResourceARNList=ResourceARNList, **kw)
failures = response.get('FailedResourcesMap', {})
if not failures:
return response
errors = {}
throttles = set()
for f_arn in failures:
error_code = failures[f_arn]['ErrorCode']
if error_code == 'ThrottlingException':
throttles.add(f_arn)
elif error_code == 'ResourceNotFoundException':
continue
else:
errors[f_arn] = error_code
if errors:
raise Exception("Resource Tag Errors %s" % (errors))
if idx == max_attempts - 1:
raise Exception("Resource Tag Throttled %s" % (", ".join(throttles)))
time.sleep(delay)
ResourceARNList = list(throttles)
def test_delays_jitter(self):
for idx, i in enumerate(utils.backoff_delays(1, 256, jitter=True)):
maxv = 2 ** idx
self.assertTrue(i > 0)
self.assertTrue(i < maxv)
def test_delays(self):
self.assertEqual(
list(utils.backoff_delays(1, 256)),
[1, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0],
)
