def retryOnCloudWatchTransientError(logger=logging.root,
timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC):
""" Create a decorator for retrying a function upon CloudWatch transient
error.
:param logger: a python logger object for logging failures; defaults to the
builtin root logger
:param timeoutSec: How many seconds from time of initial call to stop retrying
:type timeoutSec: floating point
:returns: a decorator
"""
def retryFilter(e, *_args, **_kwargs):
"""Return True to permit a retry, false to re-raise the exception."""
if isinstance(e, BotoServerError):
if getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES:
return True
else:
return False
return True
return decorators.retry(timeoutSec=timeoutSec,
initialRetryDelaySec=INITIAL_RETRY_BACKOFF_SEC,
maxRetryDelaySec=MAX_RETRY_BACKOFF_SEC,
retryExceptions=RETRY_EXCEPTIONS,
retryFilter=retryFilter,
getLoggerCallback=lambda: logger,
clientLabel="retryOnCloudWatchTransientError")
def testRetryRetryExceptionExcluded(self, mockTime, mockSleep):
""" Test that retry is not triggered if raised exception is not in
retryExceptions """
self.mockSleepTime(mockTime, mockSleep)
class TestExceptionA(Exception):
pass
class TestExceptionB(Exception):
pass
retryDecorator = decorators.retry(timeoutSec=1,
initialRetryDelaySec=1,
maxRetryDelaySec=10,
retryExceptions=(TestExceptionA, ))
@retryDecorator
def testFunction():
raise TestExceptionB("Test exception")
with self.assertRaises(TestExceptionB):
testFunction()
self.assertEqual(mockSleep.call_count, 0)
def retryOnELBTransientError(logger=logging.root,
timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC):
""" Create a decorator for retrying a function upon ELB transient error.
:param logger: a python logger object for logging failures; defaults to the
builtin root logger
:param timeoutSec: How many seconds from time of initial call to stop retrying
:type timeoutSec: floating point
:returns: a decorator
"""
def retryFilter(e, *_args, **_kwargs):
"""Return True to permit a retry, false to re-raise the exception."""
if isinstance(e, BotoServerError):
if getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES:
return True
else:
return False
return True
return decorators.retry(
timeoutSec=timeoutSec,
initialRetryDelaySec=INITIAL_RETRY_BACKOFF_SEC,
maxRetryDelaySec=MAX_RETRY_BACKOFF_SEC,
retryExceptions=RETRY_EXCEPTIONS,
retryFilter=retryFilter,
getLoggerCallback=lambda: logger,
clientLabel="retryOnELBTransientError")
def testRetryRetryFilter(self, mockTime, mockSleep):
"""Test that if retryFilter is specified and exception is in
retryExceptions, retries iff retryFilter returns true."""
self.mockSleepTime(mockTime, mockSleep)
# Test with retryFilter returning True
retryDecoratorTrueFilter = decorators.retry(
timeoutSec=1,
initialRetryDelaySec=1,
maxRetryDelaySec=10,
retryExceptions=(TestParentException, ),
retryFilter=lambda _1, _2, _3: True)
@retryDecoratorTrueFilter
def testFunctionTrue():
raise TestChildException("Test exception")
with self.assertRaises(TestChildException):
testFunctionTrue()
self.assertEqual(mockSleep.call_count, 1)
# Test with retryFilter returning False
mockSleep.reset_mock()
retryDecoratorFalseFilter = decorators.retry(
timeoutSec=1,
initialRetryDelaySec=1,
maxRetryDelaySec=10,
retryExceptions=(TestParentException, ),
retryFilter=lambda _1, _2, _3: False)
@retryDecoratorFalseFilter
def testFunctionFalse():
raise TestChildException("Test exception")
with self.assertRaises(TestChildException):
testFunctionFalse()
self.assertEqual(mockSleep.call_count, 0)
def testRetryRetryFilter(self, mockTime, mockSleep):
"""Test that if retryFilter is specified and exception is in
retryExceptions, retries iff retryFilter returns true."""
self.mockSleepTime(mockTime, mockSleep)
# Test with retryFilter returning True
retryDecoratorTrueFilter = decorators.retry(
timeoutSec=1, initialRetryDelaySec=1,
maxRetryDelaySec=10, retryExceptions=(TestParentException,),
retryFilter=lambda _1, _2, _3: True)
@retryDecoratorTrueFilter
def testFunctionTrue():
raise TestChildException("Test exception")
with self.assertRaises(TestChildException):
testFunctionTrue()
self.assertEqual(mockSleep.call_count, 1)
# Test with retryFilter returning False
mockSleep.reset_mock()
retryDecoratorFalseFilter = decorators.retry(
timeoutSec=1, initialRetryDelaySec=1,
maxRetryDelaySec=10, retryExceptions=(TestParentException,),
retryFilter=lambda _1, _2, _3: False)
@retryDecoratorFalseFilter
def testFunctionFalse():
raise TestChildException("Test exception")
with self.assertRaises(TestChildException):
testFunctionFalse()
self.assertEqual(mockSleep.call_count, 0)
def testNoRetryIfCallSucceeds(self, mockTime, mockSleep):
"""If the initial call succeeds, test that no retries are performed."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(timeoutSec=30,
initialRetryDelaySec=2,
maxRetryDelaySec=10)
testFunction = Mock(__name__="testFunction", autospec=True)
retryDecorator(testFunction)()
testFunction.assert_called_once_with()
def testNoRetryIfCallSucceeds(self, mockTime, mockSleep):
"""If the initial call succeeds, test that no retries are performed."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(
timeoutSec=30, initialRetryDelaySec=2,
maxRetryDelaySec=10)
testFunction = Mock(__name__="testFunction", autospec=True)
retryDecorator(testFunction)()
testFunction.assert_called_once_with()
def testReturnsExpectedWithExpectedArgs(self, mockTime, mockSleep):
"""Test that docorated function receives only expected args and
that it returns the expected value on success."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(
timeoutSec=30, initialRetryDelaySec=2,
maxRetryDelaySec=10)
testFunction = Mock(return_value=321,
__name__="testFunction", autospec=True)
returnValue = retryDecorator(testFunction)(1, 2, a=3, b=4)
self.assertEqual(returnValue, 321)
testFunction.assert_called_once_with(1, 2, a=3, b=4)
def testRetryRetryExceptionIncluded(self, mockTime, mockSleep):
"""Test that retry is triggered if raised exception is in
retryExceptions."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(
timeoutSec=1, initialRetryDelaySec=1,
maxRetryDelaySec=10, retryExceptions=(TestParentException,))
@retryDecorator
def testFunction():
raise TestChildException("Test exception")
with self.assertRaises(TestChildException):
testFunction()
self.assertEqual(mockSleep.call_count, 1)
def testRetryWaitsInitialRetryDelaySec(self, mockTime, mockSleep):
"""Test that delay times are correct."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(
timeoutSec=30, initialRetryDelaySec=2,
maxRetryDelaySec=10)
testFunction = Mock(side_effect=TestParentException("Test exception"),
__name__="testFunction", autospec=True)
with self.assertRaises(TestParentException):
retryDecorator(testFunction)()
self.assertEqual(mockSleep.mock_calls, [call(2), call(4), call(8),
call(10), call(10)])
self.assertEqual(testFunction.call_count, 6)
def testRetryNoTimeForRetries(self, mockTime, mockSleep):
"""Test that when timeoutSec == 0, function is executed exactly once
with no retries, and raises an exception on failure.
"""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(
timeoutSec=0, initialRetryDelaySec=0.2,
maxRetryDelaySec=10)
testFunction = Mock(side_effect=TestParentException("Test exception"),
__name__="testFunction", autospec=True)
with self.assertRaises(TestParentException):
retryDecorator(testFunction)()
self.assertFalse(mockSleep.called)
testFunction.assert_called_once_with()
def testFailsFirstSucceedsLater(self, mockTime, mockSleep):
"""If initial attempts fail but subsequent attempt succeeds, ensure that
expected number of retries is performed and expected value is returned."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(timeoutSec=30,
initialRetryDelaySec=2,
maxRetryDelaySec=10)
testFunction = Mock(side_effect=[
TestParentException("Test exception 1"),
TestParentException("Test exception 2"), 321
],
__name__="testFunction",
autospec=True)
returnValue = retryDecorator(testFunction)()
self.assertEqual(returnValue, 321)
self.assertEqual(testFunction.call_count, 3)
def testFailsFirstSucceedsLater(self, mockTime, mockSleep):
"""If initial attempts fail but subsequent attempt succeeds, ensure that
expected number of retries is performed and expected value is returned."""
self.mockSleepTime(mockTime, mockSleep)
retryDecorator = decorators.retry(
timeoutSec=30, initialRetryDelaySec=2,
maxRetryDelaySec=10)
testFunction = Mock(
side_effect=[
TestParentException("Test exception 1"),
TestParentException("Test exception 2"),
321
],
__name__="testFunction", autospec=True)
returnValue = retryDecorator(testFunction)()
self.assertEqual(returnValue, 321)
self.assertEqual(testFunction.call_count, 3)
def retryOnBotoS3TransientError(getLoggerCallback=logging.getLogger,
timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC):
"""Return a decorator for retrying a boto S3 function on transient failures.
NOTE: please ensure that the operation being retried is idempotent.
getLoggerCallback:
user-supplied callback function that takes no args and
returns the logger instance to use for logging.
timeoutSec: How many seconds from time of initial call to stop retrying
(floating point)
"""
def retryFilter(e, args, kwargs):
"""Return True to permit a retry, false to re-raise the exception."""
if isinstance(e, BotoServerError):
if (getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES):
return True
else:
return False
return True
retryExceptions = tuple([
socket.error,
AWSConnectionError,
S3DataError,
S3CreateError,
S3ResponseError,
])
return decorators.retry(timeoutSec=timeoutSec,
initialRetryDelaySec=0.1,
maxRetryDelaySec=10,
retryExceptions=retryExceptions,
retryFilter=retryFilter,
getLoggerCallback=getLoggerCallback,
clientLabel="retryOnBotoS3TransientError")
def retryOnBotoS3TransientError(getLoggerCallback=logging.getLogger,
timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC):
"""Return a decorator for retrying a boto S3 function on transient failures.
NOTE: please ensure that the operation being retried is idempotent.
getLoggerCallback:
user-supplied callback function that takes no args and
returns the logger instance to use for logging.
timeoutSec: How many seconds from time of initial call to stop retrying
(floating point)
"""
def retryFilter(e, args, kwargs):
"""Return True to permit a retry, false to re-raise the exception."""
if isinstance(e, BotoServerError):
if (getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES):
return True
else:
return False
return True
retryExceptions = tuple([
socket.error,
AWSConnectionError,
S3DataError,
S3CreateError,
S3ResponseError,
])
return decorators.retry(
timeoutSec=timeoutSec, initialRetryDelaySec=0.1, maxRetryDelaySec=10,
retryExceptions=retryExceptions, retryFilter=retryFilter,
getLoggerCallback=getLoggerCallback,
clientLabel="retryOnBotoS3TransientError")
def testRetryRetryExceptionExcluded(self, mockTime, mockSleep):
""" Test that retry is not triggered if raised exception is not in
retryExceptions """
self.mockSleepTime(mockTime, mockSleep)
class TestExceptionA(Exception):
pass
class TestExceptionB(Exception):
pass
retryDecorator = decorators.retry(
timeoutSec=1, initialRetryDelaySec=1,
maxRetryDelaySec=10, retryExceptions=(TestExceptionA,))
@retryDecorator
def testFunction():
raise TestExceptionB("Test exception")
with self.assertRaises(TestExceptionB):
testFunction()
self.assertEqual(mockSleep.call_count, 0)
