def test_wait_on_failure_does_not_exceed_60_secs_wait(self, mock_sleep):
total_attempts = 20
for attempt in range(1, total_attempts + 1):
controlflow.wait_on_failure(
attempt,
total_attempts,
'Attempt #%s' % attempt,
# Suppress messages while we make a lot of attempts.
error_print_threshold=total_attempts + 1)
# Wait time may be between 60 and 61 secs, due to rand addition.
self.assertLessEqual(mock_sleep.call_args[0][0], 61)
def test_wait_on_failure_only_prints_after_threshold(self, mock_stderr_write,
unused_mock_sleep):
total_attempts = 5
threshold = 3
for attempt in range(1, total_attempts + 1):
controlflow.wait_on_failure(
attempt,
total_attempts,
'Attempt #%s' % attempt,
error_print_threshold=threshold)
self.assertEqual(total_attempts - threshold, mock_stderr_write.call_count)
def test_wait_on_failure_waits_exponentially(self, mock_sleep):
controlflow.wait_on_failure(1, 5, 'Backoff attempt #1')
controlflow.wait_on_failure(2, 5, 'Backoff attempt #2')
controlflow.wait_on_failure(3, 5, 'Backoff attempt #3')
sleep_calls = mock_sleep.call_args_list
self.assertGreaterEqual(sleep_calls[0][0][0], 2**1)
self.assertGreaterEqual(sleep_calls[1][0][0], 2**2)
self.assertGreaterEqual(sleep_calls[2][0][0], 2**3)
def call(service,
function,
silent_errors=False,
soft_errors=False,
throw_reasons=None,
retry_reasons=None,
**kwargs):
"""Executes a single request on a Google service function.
Args:
service: A Google service object for the desired API.
function: String, The name of a service request method to execute.
silent_errors: Bool, If True, error messages are suppressed when
encountered.
soft_errors: Bool, If True, writes non-fatal errors to stderr.
throw_reasons: A list of Google HTTP error reason strings indicating the
errors generated by this request should be re-thrown. All other HTTP
errors are consumed.
retry_reasons: A list of Google HTTP error reason strings indicating which
error should be retried, using exponential backoff techniques, when the
error reason is encountered.
**kwargs: Additional params to pass to the request method.
Returns:
A response object for the corresponding Google API call.
"""
if throw_reasons is None:
throw_reasons = []
if retry_reasons is None:
retry_reasons = []
method = getattr(service, function)
retries = 10
parameters = dict(
list(kwargs.items()) + list(GM_Globals[GM_EXTRA_ARGS_DICT].items()))
for n in range(1, retries + 1):
try:
return method(**parameters).execute()
except googleapiclient.errors.HttpError as e:
http_status, reason, message = errors.get_gapi_error_detail(
e,
soft_errors=soft_errors,
silent_errors=silent_errors,
retry_on_http_error=n < 3)
if http_status == -1:
# The error detail indicated that we should retry this request
# We'll refresh credentials and make another pass
service._http.request.credentials.refresh(
transport.create_http())
continue
if http_status == 0:
return None
is_known_error_reason = reason in [
r.value for r in errors.ErrorReason
]
if is_known_error_reason and errors.ErrorReason(
reason) in throw_reasons:
if errors.ErrorReason(
reason) in errors.ERROR_REASON_TO_EXCEPTION:
raise errors.ERROR_REASON_TO_EXCEPTION[errors.ErrorReason(
reason)](message)
raise e
if (n != retries) and (
is_known_error_reason and errors.ErrorReason(reason)
in errors.DEFAULT_RETRY_REASONS + retry_reasons):
controlflow.wait_on_failure(n, retries, reason)
continue
if soft_errors:
display.print_error(
f'{http_status}: {message} - {reason}{["", ": Giving up."][n > 1]}'
)
return None
controlflow.system_error_exit(
int(http_status), f'{http_status}: {message} - {reason}')
except google.auth.exceptions.RefreshError as e:
handle_oauth_token_error(
e, soft_errors
or errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons)
if errors.ErrorReason.SERVICE_NOT_AVAILABLE in throw_reasons:
raise errors.GapiServiceNotAvailableError(str(e))
display.print_error(
f'User {GM_Globals[GM_CURRENT_API_USER]}: {str(e)}')
return None
except ValueError as e:
if hasattr(service._http,
'cache') and service._http.cache is not None:
service._http.cache = None
continue
controlflow.system_error_exit(4, str(e))
except (httplib2.ServerNotFoundError, RuntimeError) as e:
if n != retries:
service._http.connections = {}
controlflow.wait_on_failure(n, retries, str(e))
continue
controlflow.system_error_exit(4, str(e))
except TypeError as e:
controlflow.system_error_exit(4, str(e))
def test_wait_on_failure_prints_errors(self, unused_mock_sleep):
message = 'An error message to display'
with patch.object(controlflow.sys.stderr,
'write') as mock_stderr_write:
controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
self.assertIn(message, mock_stderr_write.call_args[0][0])
def test_wait_on_failure_prints_errors(self, mock_stderr_write,
unused_mock_sleep):
message = 'An error message to display'
controlflow.wait_on_failure(1, 5, message, error_print_threshold=0)
self.assertIn(message, mock_stderr_write.call_args[0][0])