def test_retry_exceeded_reraises_connection_error(self, randint_mock, sleep_mock): randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125] responses = [requests.exceptions.ConnectionError] * 8 func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0) with pytest.raises(requests.exceptions.ConnectionError): _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert func.call_count == 8 assert func.mock_calls == [mock.call()] * 8 assert randint_mock.call_count == 7 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7 assert sleep_mock.call_count == 7 sleep_mock.assert_any_call(1.875) sleep_mock.assert_any_call(2.0) sleep_mock.assert_any_call(4.375) sleep_mock.assert_any_call(8.5) sleep_mock.assert_any_call(16.5) sleep_mock.assert_any_call(32.25) sleep_mock.assert_any_call(64.125)
def http_request(transport, method, url, data=None, headers=None, retry_strategy=_DEFAULT_RETRY_STRATEGY, **transport_kwargs): """Make an HTTP request. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests via a ``request()`` method. This method must accept an HTTP method, an upload URL, a ``data`` keyword argument and a ``headers`` keyword argument. method (str): The HTTP method for the request. url (str): The URL for the request. data (Optional[bytes]): The body of the request. headers (Mapping[str, str]): The headers for the request (``transport`` may also add additional headers). retry_strategy (~google.resumable_media.common.RetryStrategy): The strategy to use if the request fails and must be retried. transport_kwargs (Dict[str, str]): Extra keyword arguments to be passed along to ``transport.request``. Returns: ~requests.Response: The return value of ``transport.request()``. """ func = functools.partial(transport.request, method, url, data=data, headers=headers, **transport_kwargs) return _helpers.wait_and_retry(func, RequestsMixin._get_status_code, retry_strategy)
def test_success_with_retry(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] status_codes = ( http_client.INTERNAL_SERVER_ERROR, http_client.BAD_GATEWAY, http_client.SERVICE_UNAVAILABLE, http_client.NOT_FOUND, ) responses = [_make_response(status_code) for status_code in status_codes] func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy() ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert status_codes[-1] not in _helpers.RETRYABLE assert func.call_count == 4 assert func.mock_calls == [mock.call()] * 4 assert randint_mock.call_count == 3 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3 assert sleep_mock.call_count == 3 sleep_mock.assert_any_call(1.125) sleep_mock.assert_any_call(2.625) sleep_mock.assert_any_call(4.375)
def test_success_with_retry_connection_error(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] response = _make_response(http_client.NOT_FOUND) responses = [ requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, response, ] func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy() ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert func.call_count == 4 assert func.mock_calls == [mock.call()] * 4 assert randint_mock.call_count == 3 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3 assert sleep_mock.call_count == 3 sleep_mock.assert_any_call(1.125) sleep_mock.assert_any_call(2.625) sleep_mock.assert_any_call(4.375)
def test_success_with_retry_custom_delay(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] status_codes = ( http_client.INTERNAL_SERVER_ERROR, http_client.BAD_GATEWAY, http_client.SERVICE_UNAVAILABLE, http_client.NOT_FOUND, ) responses = [ _make_response(status_code) for status_code in status_codes ] func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4) ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert status_codes[-1] not in common.RETRYABLE assert func.call_count == 4 assert func.mock_calls == [mock.call()] * 4 assert randint_mock.call_count == 3 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3 assert sleep_mock.call_count == 3 sleep_mock.assert_any_call(3.125) # initial delay 3 + jitter 0.125 sleep_mock.assert_any_call( 12.625) # previous delay 3 * multiplier 4 + jitter 0.625 sleep_mock.assert_any_call( 48.375) # previous delay 12 * multiplier 4 + jitter 0.375
def http_request(transport, method, url, data=None, headers=None, retry_strategy=_DEFAULT_RETRY_STRATEGY, **transport_kwargs): """Make an HTTP request. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests via a ``request()`` method. This method must accept an HTTP method, an upload URL, a ``data`` keyword argument and a ``headers`` keyword argument. method (str): The HTTP method for the request. url (str): The URL for the request. data (Optional[bytes]): The body of the request. headers (Mapping[str, str]): The headers for the request (``transport`` may also add additional headers). retry_strategy (~google.resumable_media.common.RetryStrategy): The strategy to use if the request fails and must be retried. transport_kwargs (Dict[str, str]): Extra keyword arguments to be passed along to ``transport.request``. Returns: ~requests.Response: The return value of ``transport.request()``. """ func = functools.partial( transport.request, method, url, data=data, headers=headers, **transport_kwargs) return _helpers.wait_and_retry( func, RequestsMixin._get_status_code, retry_strategy)
def consume( self, transport, timeout=( _request_helpers._DEFAULT_CONNECT_TIMEOUT, _request_helpers._DEFAULT_READ_TIMEOUT, ), ): """Consume the resource to be downloaded. If a ``stream`` is attached to this download, then the downloaded resource will be written to the stream. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. ValueError: If the current :class:`Download` has already finished. """ method, url, payload, headers = self._prepare_request() # Wrap the request business logic in a function to be retried. def retriable_request(): # NOTE: We assume "payload is None" but pass it along anyway. result = transport.request( method, url, data=payload, headers=headers, stream=True, timeout=timeout, ) self._process_response(result) if self._stream is not None: self._write_to_stream(result) return result return _helpers.wait_and_retry(retriable_request, self._get_status_code, self._retry_strategy)
def test_success_no_retry(self): truthy = http_client.OK assert truthy not in _helpers.RETRYABLE response = _make_response(truthy) func = mock.Mock(return_value=response, spec=[]) retry_strategy = common.RetryStrategy() ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val is response func.assert_called_once_with()
def test_connection_import_error_failure(self, randint_mock, sleep_mock): randint_mock.side_effect = [125, 625, 375] response = _make_response(http_client.NOT_FOUND) responses = [ requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, requests.exceptions.ConnectionError, response, ] with mock.patch( "google.resumable_media._helpers._get_connection_error_classes", side_effect=ImportError, ): with pytest.raises(requests.exceptions.ConnectionError): func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy() _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
def transmit( self, transport, data, metadata, content_type, timeout=( _request_helpers._DEFAULT_CONNECT_TIMEOUT, _request_helpers._DEFAULT_READ_TIMEOUT, ), ): """Transmit the resource to be uploaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. data (bytes): The resource content to be uploaded. metadata (Mapping[str, str]): The resource metadata, such as an ACL list. content_type (str): The content type of the resource, e.g. a JPEG image has content type ``image/jpeg``. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ method, url, payload, headers = self._prepare_request( data, metadata, content_type) # Wrap the request business logic in a function to be retried. def retriable_request(): result = transport.request(method, url, data=payload, headers=headers, timeout=timeout) self._process_response(result) return result return _helpers.wait_and_retry(retriable_request, self._get_status_code, self._retry_strategy)
def consume_next_chunk( self, transport, timeout=( _request_helpers._DEFAULT_CONNECT_TIMEOUT, _request_helpers._DEFAULT_READ_TIMEOUT, ), ): """Consume the next chunk of the resource to be downloaded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ValueError: If the current download has finished. """ method, url, payload, headers = self._prepare_request() # Wrap the request business logic in a function to be retried. def retriable_request(): # NOTE: We assume "payload is None" but pass it along anyway. result = transport.request( method, url, data=payload, headers=headers, stream=True, timeout=timeout, ) self._process_response(result) return result return _helpers.wait_and_retry(retriable_request, self._get_status_code, self._retry_strategy)
def recover(self, transport): """Recover from a failure. This method should be used when a :class:`ResumableUpload` is in an :attr:`~ResumableUpload.invalid` state due to a request failure. This will verify the progress with the server and make sure the current upload is in a valid state before :meth:`transmit_next_chunk` can be used again. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ timeout = ( _request_helpers._DEFAULT_CONNECT_TIMEOUT, _request_helpers._DEFAULT_READ_TIMEOUT, ) method, url, payload, headers = self._prepare_recover_request() # NOTE: We assume "payload is None" but pass it along anyway. # Wrap the request business logic in a function to be retried. def retriable_request(): result = transport.request(method, url, data=payload, headers=headers, timeout=timeout) self._process_recover_response(result) return result return _helpers.wait_and_retry(retriable_request, self._get_status_code, self._retry_strategy)
def test_retry_exceeds_max_cumulative(self, randint_mock, sleep_mock): randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125] status_codes = ( http_client.SERVICE_UNAVAILABLE, http_client.GATEWAY_TIMEOUT, common.TOO_MANY_REQUESTS, http_client.INTERNAL_SERVER_ERROR, http_client.SERVICE_UNAVAILABLE, http_client.BAD_GATEWAY, http_client.GATEWAY_TIMEOUT, common.TOO_MANY_REQUESTS, ) responses = [ _make_response(status_code) for status_code in status_codes ] func = mock.Mock(side_effect=responses, spec=[]) retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0) ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy) assert ret_val == responses[-1] assert status_codes[-1] in _helpers.RETRYABLE assert func.call_count == 8 assert func.mock_calls == [mock.call()] * 8 assert randint_mock.call_count == 7 assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7 assert sleep_mock.call_count == 7 sleep_mock.assert_any_call(1.875) sleep_mock.assert_any_call(2.0) sleep_mock.assert_any_call(4.375) sleep_mock.assert_any_call(8.5) sleep_mock.assert_any_call(16.5) sleep_mock.assert_any_call(32.25) sleep_mock.assert_any_call(64.125)
def transmit_next_chunk( self, transport, timeout=( _request_helpers._DEFAULT_CONNECT_TIMEOUT, _request_helpers._DEFAULT_READ_TIMEOUT, ), ): """Transmit the next chunk of the resource to be uploaded. If the current upload was initiated with ``stream_final=False``, this method will dynamically determine if the upload has completed. The upload will be considered complete if the stream produces fewer than :attr:`chunk_size` bytes when a chunk is read from it. In the case of failure, an exception is thrown that preserves the failed response: .. testsetup:: bad-response import io import mock import requests import http.client from google import resumable_media import google.resumable_media.requests.upload as upload_mod transport = mock.Mock(spec=['request']) fake_response = requests.Response() fake_response.status_code = int(http.client.BAD_REQUEST) transport.request.return_value = fake_response upload_url = 'http://test.invalid' upload = upload_mod.ResumableUpload( upload_url, resumable_media.UPLOAD_CHUNK_SIZE) # Fake that the upload has been initiate()-d data = b'data is here' upload._stream = io.BytesIO(data) upload._total_bytes = len(data) upload._resumable_url = 'http://test.invalid?upload_id=nope' .. doctest:: bad-response :options: +NORMALIZE_WHITESPACE >>> error = None >>> try: ... upload.transmit_next_chunk(transport) ... except resumable_media.InvalidResponse as caught_exc: ... error = caught_exc ... >>> error InvalidResponse('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>, <HTTPStatus.PERMANENT_REDIRECT: 308>) >>> error.response <Response [400]> Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. Raises: ~google.resumable_media.common.InvalidResponse: If the status code is not 200 or http.client.PERMANENT_REDIRECT. ~google.resumable_media.common.DataCorruption: If this is the final chunk, a checksum validation was requested, and the checksum does not match or is not available. """ method, url, payload, headers = self._prepare_request() # Wrap the request business logic in a function to be retried. def retriable_request(): result = transport.request(method, url, data=payload, headers=headers, timeout=timeout) self._process_response(result, len(payload)) return result return _helpers.wait_and_retry(retriable_request, self._get_status_code, self._retry_strategy)
def initiate( self, transport, stream, metadata, content_type, total_bytes=None, stream_final=True, timeout=( _request_helpers._DEFAULT_CONNECT_TIMEOUT, _request_helpers._DEFAULT_READ_TIMEOUT, ), ): """Initiate a resumable upload. By default, this method assumes your ``stream`` is in a "final" state ready to transmit. However, ``stream_final=False`` can be used to indicate that the size of the resource is not known. This can happen if bytes are being dynamically fed into ``stream``, e.g. if the stream is attached to application logs. If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be read from the stream every time :meth:`transmit_next_chunk` is called. If one of those reads produces strictly fewer bites than the chunk size, the upload will be concluded. Args: transport (~requests.Session): A ``requests`` object which can make authenticated requests. stream (IO[bytes]): The stream (i.e. file-like object) that will be uploaded. The stream **must** be at the beginning (i.e. ``stream.tell() == 0``). metadata (Mapping[str, str]): The resource metadata, such as an ACL list. content_type (str): The content type of the resource, e.g. a JPEG image has content type ``image/jpeg``. total_bytes (Optional[int]): The total number of bytes to be uploaded. If specified, the upload size **will not** be determined from the stream (even if ``stream_final=True``). stream_final (Optional[bool]): Indicates if the ``stream`` is "final" (i.e. no more bytes will be added to it). In this case we determine the upload size from the size of the stream. If ``total_bytes`` is passed, this argument will be ignored. timeout (Optional[Union[float, Tuple[float, float]]]): The number of seconds to wait for the server response. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. Returns: ~requests.Response: The HTTP response returned by ``transport``. """ method, url, payload, headers = self._prepare_initiate_request( stream, metadata, content_type, total_bytes=total_bytes, stream_final=stream_final, ) # Wrap the request business logic in a function to be retried. def retriable_request(): result = transport.request(method, url, data=payload, headers=headers, timeout=timeout) self._process_initiate_response(result) return result return _helpers.wait_and_retry(retriable_request, self._get_status_code, self._retry_strategy)