def retryOnCloudWatchTransientError(logger=logging.root, timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC): """ Create a decorator for retrying a function upon CloudWatch transient error. :param logger: a python logger object for logging failures; defaults to the builtin root logger :param timeoutSec: How many seconds from time of initial call to stop retrying :type timeoutSec: floating point :returns: a decorator """ def retryFilter(e, *_args, **_kwargs): """Return True to permit a retry, false to re-raise the exception.""" if isinstance(e, BotoServerError): if getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES: return True else: return False return True return decorators.retry(timeoutSec=timeoutSec, initialRetryDelaySec=INITIAL_RETRY_BACKOFF_SEC, maxRetryDelaySec=MAX_RETRY_BACKOFF_SEC, retryExceptions=RETRY_EXCEPTIONS, retryFilter=retryFilter, getLoggerCallback=lambda: logger, clientLabel="retryOnCloudWatchTransientError")
def testRetryRetryExceptionExcluded(self, mockTime, mockSleep): """ Test that retry is not triggered if raised exception is not in retryExceptions """ self.mockSleepTime(mockTime, mockSleep) class TestExceptionA(Exception): pass class TestExceptionB(Exception): pass retryDecorator = decorators.retry(timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestExceptionA, )) @retryDecorator def testFunction(): raise TestExceptionB("Test exception") with self.assertRaises(TestExceptionB): testFunction() self.assertEqual(mockSleep.call_count, 0)
def retryOnELBTransientError(logger=logging.root, timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC): """ Create a decorator for retrying a function upon ELB transient error. :param logger: a python logger object for logging failures; defaults to the builtin root logger :param timeoutSec: How many seconds from time of initial call to stop retrying :type timeoutSec: floating point :returns: a decorator """ def retryFilter(e, *_args, **_kwargs): """Return True to permit a retry, false to re-raise the exception.""" if isinstance(e, BotoServerError): if getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES: return True else: return False return True return decorators.retry( timeoutSec=timeoutSec, initialRetryDelaySec=INITIAL_RETRY_BACKOFF_SEC, maxRetryDelaySec=MAX_RETRY_BACKOFF_SEC, retryExceptions=RETRY_EXCEPTIONS, retryFilter=retryFilter, getLoggerCallback=lambda: logger, clientLabel="retryOnELBTransientError")
def testRetryRetryFilter(self, mockTime, mockSleep): """Test that if retryFilter is specified and exception is in retryExceptions, retries iff retryFilter returns true.""" self.mockSleepTime(mockTime, mockSleep) # Test with retryFilter returning True retryDecoratorTrueFilter = decorators.retry( timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestParentException, ), retryFilter=lambda _1, _2, _3: True) @retryDecoratorTrueFilter def testFunctionTrue(): raise TestChildException("Test exception") with self.assertRaises(TestChildException): testFunctionTrue() self.assertEqual(mockSleep.call_count, 1) # Test with retryFilter returning False mockSleep.reset_mock() retryDecoratorFalseFilter = decorators.retry( timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestParentException, ), retryFilter=lambda _1, _2, _3: False) @retryDecoratorFalseFilter def testFunctionFalse(): raise TestChildException("Test exception") with self.assertRaises(TestChildException): testFunctionFalse() self.assertEqual(mockSleep.call_count, 0)
def testRetryRetryFilter(self, mockTime, mockSleep): """Test that if retryFilter is specified and exception is in retryExceptions, retries iff retryFilter returns true.""" self.mockSleepTime(mockTime, mockSleep) # Test with retryFilter returning True retryDecoratorTrueFilter = decorators.retry( timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestParentException,), retryFilter=lambda _1, _2, _3: True) @retryDecoratorTrueFilter def testFunctionTrue(): raise TestChildException("Test exception") with self.assertRaises(TestChildException): testFunctionTrue() self.assertEqual(mockSleep.call_count, 1) # Test with retryFilter returning False mockSleep.reset_mock() retryDecoratorFalseFilter = decorators.retry( timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestParentException,), retryFilter=lambda _1, _2, _3: False) @retryDecoratorFalseFilter def testFunctionFalse(): raise TestChildException("Test exception") with self.assertRaises(TestChildException): testFunctionFalse() self.assertEqual(mockSleep.call_count, 0)
def testNoRetryIfCallSucceeds(self, mockTime, mockSleep): """If the initial call succeeds, test that no retries are performed.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry(timeoutSec=30, initialRetryDelaySec=2, maxRetryDelaySec=10) testFunction = Mock(__name__="testFunction", autospec=True) retryDecorator(testFunction)() testFunction.assert_called_once_with()
def testNoRetryIfCallSucceeds(self, mockTime, mockSleep): """If the initial call succeeds, test that no retries are performed.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry( timeoutSec=30, initialRetryDelaySec=2, maxRetryDelaySec=10) testFunction = Mock(__name__="testFunction", autospec=True) retryDecorator(testFunction)() testFunction.assert_called_once_with()
def testReturnsExpectedWithExpectedArgs(self, mockTime, mockSleep): """Test that docorated function receives only expected args and that it returns the expected value on success.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry( timeoutSec=30, initialRetryDelaySec=2, maxRetryDelaySec=10) testFunction = Mock(return_value=321, __name__="testFunction", autospec=True) returnValue = retryDecorator(testFunction)(1, 2, a=3, b=4) self.assertEqual(returnValue, 321) testFunction.assert_called_once_with(1, 2, a=3, b=4)
def testRetryRetryExceptionIncluded(self, mockTime, mockSleep): """Test that retry is triggered if raised exception is in retryExceptions.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry( timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestParentException,)) @retryDecorator def testFunction(): raise TestChildException("Test exception") with self.assertRaises(TestChildException): testFunction() self.assertEqual(mockSleep.call_count, 1)
def testRetryWaitsInitialRetryDelaySec(self, mockTime, mockSleep): """Test that delay times are correct.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry( timeoutSec=30, initialRetryDelaySec=2, maxRetryDelaySec=10) testFunction = Mock(side_effect=TestParentException("Test exception"), __name__="testFunction", autospec=True) with self.assertRaises(TestParentException): retryDecorator(testFunction)() self.assertEqual(mockSleep.mock_calls, [call(2), call(4), call(8), call(10), call(10)]) self.assertEqual(testFunction.call_count, 6)
def testRetryNoTimeForRetries(self, mockTime, mockSleep): """Test that when timeoutSec == 0, function is executed exactly once with no retries, and raises an exception on failure. """ self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry( timeoutSec=0, initialRetryDelaySec=0.2, maxRetryDelaySec=10) testFunction = Mock(side_effect=TestParentException("Test exception"), __name__="testFunction", autospec=True) with self.assertRaises(TestParentException): retryDecorator(testFunction)() self.assertFalse(mockSleep.called) testFunction.assert_called_once_with()
def testFailsFirstSucceedsLater(self, mockTime, mockSleep): """If initial attempts fail but subsequent attempt succeeds, ensure that expected number of retries is performed and expected value is returned.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry(timeoutSec=30, initialRetryDelaySec=2, maxRetryDelaySec=10) testFunction = Mock(side_effect=[ TestParentException("Test exception 1"), TestParentException("Test exception 2"), 321 ], __name__="testFunction", autospec=True) returnValue = retryDecorator(testFunction)() self.assertEqual(returnValue, 321) self.assertEqual(testFunction.call_count, 3)
def testFailsFirstSucceedsLater(self, mockTime, mockSleep): """If initial attempts fail but subsequent attempt succeeds, ensure that expected number of retries is performed and expected value is returned.""" self.mockSleepTime(mockTime, mockSleep) retryDecorator = decorators.retry( timeoutSec=30, initialRetryDelaySec=2, maxRetryDelaySec=10) testFunction = Mock( side_effect=[ TestParentException("Test exception 1"), TestParentException("Test exception 2"), 321 ], __name__="testFunction", autospec=True) returnValue = retryDecorator(testFunction)() self.assertEqual(returnValue, 321) self.assertEqual(testFunction.call_count, 3)
def retryOnBotoS3TransientError(getLoggerCallback=logging.getLogger, timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC): """Return a decorator for retrying a boto S3 function on transient failures. NOTE: please ensure that the operation being retried is idempotent. getLoggerCallback: user-supplied callback function that takes no args and returns the logger instance to use for logging. timeoutSec: How many seconds from time of initial call to stop retrying (floating point) """ def retryFilter(e, args, kwargs): """Return True to permit a retry, false to re-raise the exception.""" if isinstance(e, BotoServerError): if (getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES): return True else: return False return True retryExceptions = tuple([ socket.error, AWSConnectionError, S3DataError, S3CreateError, S3ResponseError, ]) return decorators.retry(timeoutSec=timeoutSec, initialRetryDelaySec=0.1, maxRetryDelaySec=10, retryExceptions=retryExceptions, retryFilter=retryFilter, getLoggerCallback=getLoggerCallback, clientLabel="retryOnBotoS3TransientError")
def retryOnBotoS3TransientError(getLoggerCallback=logging.getLogger, timeoutSec=DEFAULT_RETRY_TIMEOUT_SEC): """Return a decorator for retrying a boto S3 function on transient failures. NOTE: please ensure that the operation being retried is idempotent. getLoggerCallback: user-supplied callback function that takes no args and returns the logger instance to use for logging. timeoutSec: How many seconds from time of initial call to stop retrying (floating point) """ def retryFilter(e, args, kwargs): """Return True to permit a retry, false to re-raise the exception.""" if isinstance(e, BotoServerError): if (getattr(e, "error_code", "") in RETRIABLE_SERVER_ERROR_CODES): return True else: return False return True retryExceptions = tuple([ socket.error, AWSConnectionError, S3DataError, S3CreateError, S3ResponseError, ]) return decorators.retry( timeoutSec=timeoutSec, initialRetryDelaySec=0.1, maxRetryDelaySec=10, retryExceptions=retryExceptions, retryFilter=retryFilter, getLoggerCallback=getLoggerCallback, clientLabel="retryOnBotoS3TransientError")
def testRetryRetryExceptionExcluded(self, mockTime, mockSleep): """ Test that retry is not triggered if raised exception is not in retryExceptions """ self.mockSleepTime(mockTime, mockSleep) class TestExceptionA(Exception): pass class TestExceptionB(Exception): pass retryDecorator = decorators.retry( timeoutSec=1, initialRetryDelaySec=1, maxRetryDelaySec=10, retryExceptions=(TestExceptionA,)) @retryDecorator def testFunction(): raise TestExceptionB("Test exception") with self.assertRaises(TestExceptionB): testFunction() self.assertEqual(mockSleep.call_count, 0)