def inner(request, options=None):
"""Invoke with the actual settings."""
this_settings = settings.merge(options)
if this_settings.retry and this_settings.retry.retry_codes:
api_call = retry.retryable(func, this_settings.retry,
**this_settings.kwargs)
else:
api_call = retry.add_timeout_arg(func, this_settings.timeout,
**this_settings.kwargs)
api_call = _catch_errors(api_call, config.API_ERRORS)
return api_caller(api_call, this_settings, request)
def test_retryable_with_timeout(self, mock_time, mock_exc_to_code):
mock_time.return_value = 1
mock_exc_to_code.side_effect = lambda e: e.code
mock_call = mock.Mock()
mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1),
mock.DEFAULT]
mock_call.return_value = 1729
retry_options = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, 0, 0, 0, 0))
my_callable = retry.retryable(mock_call, retry_options)
self.assertRaises(errors.RetryError, my_callable)
self.assertEqual(0, mock_call.call_count)
def test_retryable_without_timeout(self, mock_time, mock_exc_to_code):
mock_time.return_value = 0
mock_exc_to_code.side_effect = lambda e: e.code
to_attempt = 3
mock_call = mock.Mock()
mock_call.side_effect = ([CustomException('', _FAKE_STATUS_CODE_1)] *
(to_attempt - 1) + [mock.DEFAULT])
mock_call.return_value = 1729
retry_options = RetryOptions(
[_FAKE_STATUS_CODE_1],
BackoffSettings(0, 0, 0, None, None, None, None))
my_callable = retry.retryable(mock_call, retry_options)
self.assertEqual(my_callable(None), 1729)
self.assertEqual(to_attempt, mock_call.call_count)
def _poll(self, timeout=None):
def _done_check(_):
# Check exceptional case: raise if in progress
if not self.done():
raise _DeadlineExceededError()
# Return expected operation
return self._operation
# If a timeout is set, then convert it to milliseconds.
#
# Also, we need to send 0 instead of None for the rpc arguments,
# because an internal method (`_has_timeout_settings`) will
# erroneously return False otherwise.
# TODO (lukesneeringer): Look into this.
rpc_arg = None
if timeout is not None:
timeout *= 1000
rpc_arg = 0
# Set the backoff settings. We have specific backoff settings
# for "are we there yet" calls that are distinct from those configured
# in the config.json files.
backoff_settings = BackoffSettings(
initial_retry_delay_millis=1000,
retry_delay_multiplier=2,
max_retry_delay_millis=30000,
initial_rpc_timeout_millis=rpc_arg,
rpc_timeout_multiplier=rpc_arg,
max_rpc_timeout_millis=rpc_arg,
total_timeout_millis=timeout,
)
# Set the retry to retry if `_done_check` raises the
# _DeadlineExceededError, according to the given backoff settings.
retry_options = RetryOptions([StatusCode.DEADLINE_EXCEEDED],
backoff_settings)
retryable_done_check = retryable(_done_check, retry_options)
# Start polling, and return the final result from `_done_check`.
return retryable_done_check()
def _poll(self, timeout=None):
def _done_check(_):
# Check exceptional case: raise if in progress
if not self.done():
raise _DeadlineExceededError()
# Return expected operation
return self._operation
if timeout is None:
backoff_settings = BackoffSettings(1000, 2, 30000, None, None,
None, None)
else:
backoff_settings = BackoffSettings(1000, 2, 30000, 0, 0, 0,
timeout * _MILLIS_PER_SEC)
retry_options = RetryOptions([StatusCode.DEADLINE_EXCEEDED],
backoff_settings)
retryable_done_check = retryable(_done_check, retry_options)
return retryable_done_check()
def _poll(self, timeout=None):
def _done_check(_):
# Check exceptional case: raise if in progress
if not self.done():
raise _DeadlineExceededError()
# Return expected operation
return self._operation
# If a timeout is set, then convert it to milliseconds.
#
# Also, we need to send 0 instead of None for the rpc arguments,
# because an internal method (`_has_timeout_settings`) will
# erroneously return False otherwise.
rpc_arg = None
if timeout is not None:
timeout *= 1000
rpc_arg = 0
# Set the backoff settings. We have specific backoff settings
# for "are we there yet" calls that are distinct from those configured
# in the config.json files.
backoff_settings = BackoffSettings(
initial_retry_delay_millis=1000,
retry_delay_multiplier=2,
max_retry_delay_millis=30000,
initial_rpc_timeout_millis=rpc_arg,
rpc_timeout_multiplier=rpc_arg,
max_rpc_timeout_millis=rpc_arg,
total_timeout_millis=timeout,
)
# Set the retry to retry if `_done_check` raises the
# _DeadlineExceededError, according to the given backoff settings.
retry_options = RetryOptions(
[StatusCode.DEADLINE_EXCEEDED], backoff_settings)
retryable_done_check = retryable(_done_check, retry_options)
# Start polling, and return the final result from `_done_check`.
return retryable_done_check()
def test_retryable_when_no_codes(self, mock_time, mock_exc_to_code):
mock_time.return_value = 0
mock_exc_to_code.side_effect = lambda e: e.code
mock_call = mock.Mock()
mock_call.side_effect = [CustomException('', _FAKE_STATUS_CODE_1),
mock.DEFAULT]
mock_call.return_value = 1729
retry_options = RetryOptions(
[],
BackoffSettings(0, 0, 0, 0, 0, 0, 1))
my_callable = retry.retryable(mock_call, retry_options)
try:
my_callable(None)
self.fail('Should not have been reached')
except errors.RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertEqual(1, mock_call.call_count)
def test_retryable_exponential_backoff(
self, mock_time, mock_sleep, mock_exc_to_code):
def incr_time(secs):
mock_time.return_value += secs
def api_call(timeout):
incr_time(timeout)
raise CustomException(str(timeout), _FAKE_STATUS_CODE_1)
mock_time.return_value = 0
mock_sleep.side_effect = incr_time
mock_exc_to_code.side_effect = lambda e: e.code
mock_call = mock.Mock()
mock_call.side_effect = api_call
params = BackoffSettings(3, 2, 24, 5, 2, 80, 2500)
retry_options = RetryOptions([_FAKE_STATUS_CODE_1], params)
my_callable = retry.retryable(mock_call, retry_options)
try:
my_callable()
self.fail('Should not have been reached')
except errors.RetryError as exc:
self.assertIsInstance(exc.cause, CustomException)
self.assertGreaterEqual(mock_time(),
params.total_timeout_millis / _MILLIS_PER_SEC)
# Very rough bounds
calls_lower_bound = params.total_timeout_millis / (
params.max_retry_delay_millis + params.max_rpc_timeout_millis)
self.assertGreater(mock_call.call_count, calls_lower_bound)
calls_upper_bound = (params.total_timeout_millis /
params.initial_retry_delay_millis)
self.assertLess(mock_call.call_count, calls_upper_bound)