def test_constructor_custom_delay_and_multiplier(self): retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4) assert retry_strategy.max_sleep == common.MAX_SLEEP assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY assert retry_strategy.max_retries is None assert retry_strategy.initial_delay == 3.0 assert retry_strategy.multiplier == 4
def __init__(self, upload_url, headers=None): self.upload_url = upload_url if headers is None: headers = {} self._headers = headers self._finished = False self._retry_strategy = common.RetryStrategy()
async def test_retry_exceeded_reraises_connection_error( self, randint_mock, sleep_mock ): randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125] responses = [ConnectionError] * 8 func = mock.AsyncMock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0) with pytest.raises(ConnectionError): await _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert func.call_count == 8 assert func.mock_calls == [mock.call()] * 8 assert randint_mock.call_count == 7 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7 assert sleep_mock.call_count == 7 sleep_mock.assert_any_call(1.875) sleep_mock.assert_any_call(2.0) sleep_mock.assert_any_call(4.375) sleep_mock.assert_any_call(8.5) sleep_mock.assert_any_call(16.5) sleep_mock.assert_any_call(32.25) sleep_mock.assert_any_call(64.125)
def test_success_with_retry_connection_error(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] response = _make_response(http_client.NOT_FOUND) responses = [ requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, response, ] func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy() ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert func.call_count == 4 assert func.mock_calls == [mock.call()] * 4 assert randint_mock.call_count == 3 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3 assert sleep_mock.call_count == 3 sleep_mock.assert_any_call(1.125) sleep_mock.assert_any_call(2.625) sleep_mock.assert_any_call(4.375)
async def test_success_with_retry(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] status_codes = ( http_client.INTERNAL_SERVER_ERROR, http_client.BAD_GATEWAY, http_client.SERVICE_UNAVAILABLE, http_client.NOT_FOUND, ) responses = [_make_response(status_code) for status_code in status_codes] func = mock.AsyncMock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy() ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert status_codes[-1] not in common.RETRYABLE assert func.call_count == 4 assert func.mock_calls == [mock.call()] * 4 assert randint_mock.call_count == 3 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3 assert sleep_mock.call_count == 3 sleep_mock.assert_any_call(1.125) sleep_mock.assert_any_call(2.625) sleep_mock.assert_any_call(4.375)
def test_success_with_retry_custom_delay(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] status_codes = ( http_client.INTERNAL_SERVER_ERROR, http_client.BAD_GATEWAY, http_client.SERVICE_UNAVAILABLE, http_client.NOT_FOUND, ) responses = [ _make_response(status_code) for status_code in status_codes ] func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4) ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert status_codes[-1] not in common.RETRYABLE assert func.call_count == 4 assert func.mock_calls == [mock.call()] * 4 assert randint_mock.call_count == 3 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3 assert sleep_mock.call_count == 3 sleep_mock.assert_any_call(3.125) # initial delay 3 + jitter 0.125 sleep_mock.assert_any_call( 12.625) # previous delay 3 * multiplier 4 + jitter 0.625 sleep_mock.assert_any_call( 48.375) # previous delay 12 * multiplier 4 + jitter 0.375
def test_retry_allowed_bound_cumulative(self): retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0) assert retry_strategy.retry_allowed(50.0, 10) assert retry_strategy.retry_allowed(99.0, 7) assert retry_strategy.retry_allowed(100.0, 4) assert not retry_strategy.retry_allowed(101.0, 11) assert not retry_strategy.retry_allowed(200.0, 6)
def test_constructor_explicit_bound_retries(self): max_sleep = 13.75 max_retries = 14 retry_strategy = common.RetryStrategy(max_sleep=max_sleep, max_retries=max_retries) assert retry_strategy.max_sleep == max_sleep assert retry_strategy.max_cumulative_retry is None assert retry_strategy.max_retries == max_retries
def test_constructor_explicit_bound_cumulative(self): max_sleep = 10.0 max_cumulative_retry = 100.0 retry_strategy = common.RetryStrategy( max_sleep=max_sleep, max_cumulative_retry=max_cumulative_retry) assert retry_strategy.max_sleep == max_sleep assert retry_strategy.max_cumulative_retry == max_cumulative_retry assert retry_strategy.max_retries is None
def __init__(self, media_url, start=None, end=None, headers=None): self.media_url = media_url self.start = start self.end = end if headers is None: headers = {} self._headers = headers self._finished = False self._retry_strategy = common.RetryStrategy()
async def test_success_no_retry(self): truthy = http_client.OK assert truthy not in common.RETRYABLE response = _make_response(truthy) func = mock.AsyncMock(return_value=response, spec=[]) retry_strategy = common.RetryStrategy() ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val is response func.assert_called_once_with()
def test_connection_import_error_failure(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] response = _make_response(http_client.NOT_FOUND) responses = [ requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, response, ] with mock.patch( "google.resumable_media._helpers._get_connection_error_classes", side_effect=ImportError, ): with pytest.raises(requests.exceptions.ConnectionError): func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy() _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
async def test_retry_exceeds_max_cumulative(self, randint_mock, sleep_mock): randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125] status_codes = ( http_client.SERVICE_UNAVAILABLE, http_client.GATEWAY_TIMEOUT, common.TOO_MANY_REQUESTS, http_client.INTERNAL_SERVER_ERROR, http_client.SERVICE_UNAVAILABLE, http_client.BAD_GATEWAY, http_client.GATEWAY_TIMEOUT, common.TOO_MANY_REQUESTS, ) responses = [ _make_response(status_code) for status_code in status_codes ] func = mock.AsyncMock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0) ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert status_codes[-1] in _helpers.RETRYABLE assert func.call_count == 8 assert func.mock_calls == [mock.call()] * 8 assert randint_mock.call_count == 7 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7 assert sleep_mock.call_count == 7 sleep_mock.assert_any_call(1.875) sleep_mock.assert_any_call(2.0) sleep_mock.assert_any_call(4.375) sleep_mock.assert_any_call(8.5) sleep_mock.assert_any_call(16.5) sleep_mock.assert_any_call(32.25) sleep_mock.assert_any_call(64.125)
def test_constructor_failure(self): with pytest.raises(ValueError) as exc_info: common.RetryStrategy(max_cumulative_retry=600.0, max_retries=12) exc_info.match(common._SLEEP_RETRY_ERROR_MSG)
def test_constructor_defaults(self): retry_strategy = common.RetryStrategy() assert retry_strategy.max_sleep == common.MAX_SLEEP assert ( retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY) assert retry_strategy.max_retries is None
def test_retry_allowed_bound_retries(self): retry_strategy = common.RetryStrategy(max_retries=6) assert retry_strategy.retry_allowed(1000.0, 5) assert retry_strategy.retry_allowed(99.0, 6) assert not retry_strategy.retry_allowed(625.5, 7)
"""Shared utilities used by both downloads and uploads. This utilities are explicitly catered to ``requests``-like transports. """ import functools from google._async_resumable_media import _helpers from google.resumable_media import common import google.auth.transport._aiohttp_requests as aiohttp_requests import aiohttp _DEFAULT_RETRY_STRATEGY = common.RetryStrategy() _SINGLE_GET_CHUNK_SIZE = 8192 # The number of seconds to wait to establish a connection # (connect() call on socket). Avoid setting this to a multiple of 3 to not # Align with TCP Retransmission timing. (typically 2.5-3s) _DEFAULT_CONNECT_TIMEOUT = 61 # The number of seconds to wait between bytes sent from the server. _DEFAULT_READ_TIMEOUT = 60 _DEFAULT_TIMEOUT = aiohttp.ClientTimeout( connect=_DEFAULT_CONNECT_TIMEOUT, sock_read=_DEFAULT_READ_TIMEOUT ) class RequestsMixin(object):