def test_broken_pipe_retries(self, verbose):
if sys.version_info.major == 3:
# To construct a BrokenPipeError, we pass errno.EPIPE (32) to OSError
# See https://docs.python.org/3/library/exceptions.html#BaseException.args
exception_list = [
OSError(errno.EPIPE, 'broken pipe exception test'),
OSError(errno.EPIPE, 'broken pipe exception test'),
]
elif sys.version_info.major == 2:
# In Python2, BrokenPipeErrors are socket errors
broken_pipe_socket_error = socket.error()
broken_pipe_socket_error.errno = errno.EPIPE
exception_list = [
broken_pipe_socket_error,
broken_pipe_socket_error,
]
else:
raise RuntimeError('Python version {} is not supported'.format(
sys.version_info.major))
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry twice, for a total of 1 + 2 = 3 seconds
self.assertEqual(mock_api_object.retry_counter, 2)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 3)
self.assertLess(elapsed_time_in_seconds(ft), 3.5)
def test_success(self, verbose):
exception_list = []
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
start = current_time_ms()
api_wrapper_to_test.execute(mock_api_object)
# No expected retries, and no expected wait time
self.assertEqual(mock_api_object.retry_counter, 0)
self.assertLess(current_time_ms() - start, 500)
def test_success(self, verbose):
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
exception_list = []
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# No expected retries, and no expected wait time
self.assertEqual(mock_api_object.retry_counter, 0)
self.assertLess(elapsed_time_in_seconds(ft), 1)
def test_retry_once(self, verbose, error_code):
exception_list = [
apiclient.errors.HttpError(ResponseMock(error_code, None),
b'test_exception'),
]
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
start = current_time_ms()
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry once, for about 1 second
self.assertEqual(mock_api_object.retry_counter, 1)
self.assertGreaterEqual(current_time_ms() - start, 1000)
self.assertLess(current_time_ms() - start, 1500)
def test_socket_timeout(self):
exception_list = [
socket.timeout(),
socket.timeout(),
]
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
api_wrapper_to_test = google_base.Api()
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry twice, for a total of 1 + 2 = 3 seconds
self.assertEqual(mock_api_object.retry_counter, 2)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 3)
self.assertLess(elapsed_time_in_seconds(ft), 3.5)
def test_retry_once(self, verbose, error_code):
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
exception_list = [
apiclient.errors.HttpError(ResponseMock(error_code, None),
b'test_exception'),
]
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry once, for about 1 second
self.assertEqual(mock_api_object.retry_counter, 1)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 1)
self.assertLess(elapsed_time_in_seconds(ft), 1.5)
def test_transient_retries(self, verbose):
exception_list = [
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(503, None),
b'test_exception'),
]
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
start = current_time_ms()
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry twice, for a total of 1 + 2 = 3 seconds
self.assertEqual(mock_api_object.retry_counter, 2)
self.assertGreaterEqual(current_time_ms() - start, 3000)
self.assertLess(current_time_ms() - start, 3500)
def test_broken_pipe_retries(self):
# To construct a BrokenPipeError, we pass errno.EPIPE (32) to OSError
# See https://docs.python.org/3/library/exceptions.html#BaseException.args
exception_list = [
OSError(errno.EPIPE, 'broken pipe exception test'),
OSError(errno.EPIPE, 'broken pipe exception test'),
]
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
api_wrapper_to_test = google_base.Api()
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry twice, for a total of 1 + 2 = 3 seconds
self.assertEqual(mock_api_object.retry_counter, 2)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 3)
self.assertLess(elapsed_time_in_seconds(ft), 3.5)
def test_transient_retries(self):
exception_list = [
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(503, None),
b'test_exception'),
]
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
api_wrapper_to_test = google_base.Api()
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry twice, for a total of 1 + 2 = 3 seconds
self.assertEqual(mock_api_object.retry_counter, 2)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 3)
self.assertLess(elapsed_time_in_seconds(ft), 3.5)
def test_auth_failure(self, verbose):
exception_list = [
apiclient.errors.HttpError(ResponseMock(403, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(401, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(403, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(401, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(403, None),
b'test_exception'),
]
api_wrapper_to_test = google_base.Api(verbose)
mock_api_object = GoogleApiMock(exception_list)
# We don't want to retry auth errors as aggressively, so we expect
# this exception to be raised after 4 retries,
# for a total of 1 + 2 + 4 + 8 = 15 seconds
start = current_time_ms()
with self.assertRaises(apiclient.errors.HttpError):
api_wrapper_to_test.execute(mock_api_object)
self.assertGreaterEqual(current_time_ms() - start, 15000)
self.assertLess(current_time_ms() - start, 15500)
def test_auth_failure(self):
exception_list = [
apiclient.errors.HttpError(ResponseMock(403, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(401, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(403, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(401, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(403, None),
b'test_exception'),
]
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
api_wrapper_to_test = google_base.Api()
mock_api_object = GoogleApiMock(exception_list)
# We don't want to retry auth errors as aggressively, so we expect
# this exception to be raised after 4 retries,
# for a total of 1 + 2 + 4 + 8 = 15 seconds
with self.assertRaises(apiclient.errors.HttpError):
api_wrapper_to_test.execute(mock_api_object)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 15)
self.assertLess(elapsed_time_in_seconds(ft), 15.5)
def test_retry_then_succeed(self):
exception_list = [
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
apiclient.errors.HttpError(ResponseMock(500, None),
b'test_exception'),
]
ft = fake_time.FakeTime(chronology())
with patch('time.sleep', new=ft.sleep):
api_wrapper_to_test = google_base.Api()
mock_api_object = GoogleApiMock(exception_list)
api_wrapper_to_test.execute(mock_api_object)
# Expected to retry 5 times,
# for a total of 1 + 2 + 4 + 8 + 16 = 31 seconds
# At the end we expect a recovery message to be emitted.
self.assertEqual(mock_api_object.retry_counter, 5)
self.assertGreaterEqual(elapsed_time_in_seconds(ft), 31)
self.assertLess(elapsed_time_in_seconds(ft), 31.5)
