def test_retry_exceeded_reraises_connection_error(self, randint_mock,
sleep_mock):
randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
responses = [requests.exceptions.ConnectionError] * 8
func = mock.Mock(side_effect=responses, spec=[])
retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
with pytest.raises(requests.exceptions.ConnectionError):
_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
assert func.call_count == 8
assert func.mock_calls == [mock.call()] * 8
assert randint_mock.call_count == 7
assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
assert sleep_mock.call_count == 7
sleep_mock.assert_any_call(1.875)
sleep_mock.assert_any_call(2.0)
sleep_mock.assert_any_call(4.375)
sleep_mock.assert_any_call(8.5)
sleep_mock.assert_any_call(16.5)
sleep_mock.assert_any_call(32.25)
sleep_mock.assert_any_call(64.125)
def http_request(transport,
method,
url,
data=None,
headers=None,
retry_strategy=_DEFAULT_RETRY_STRATEGY,
**transport_kwargs):
"""Make an HTTP request.
Args:
transport (~requests.Session): A ``requests`` object which can make
authenticated requests via a ``request()`` method. This method
must accept an HTTP method, an upload URL, a ``data`` keyword
argument and a ``headers`` keyword argument.
method (str): The HTTP method for the request.
url (str): The URL for the request.
data (Optional[bytes]): The body of the request.
headers (Mapping[str, str]): The headers for the request (``transport``
may also add additional headers).
retry_strategy (~google.resumable_media.common.RetryStrategy): The
strategy to use if the request fails and must be retried.
transport_kwargs (Dict[str, str]): Extra keyword arguments to be
passed along to ``transport.request``.
Returns:
~requests.Response: The return value of ``transport.request()``.
"""
func = functools.partial(transport.request,
method,
url,
data=data,
headers=headers,
**transport_kwargs)
return _helpers.wait_and_retry(func, RequestsMixin._get_status_code,
retry_strategy)
def test_success_with_retry(self, randint_mock, sleep_mock):
randint_mock.side_effect = [125, 625, 375]
status_codes = (
http_client.INTERNAL_SERVER_ERROR,
http_client.BAD_GATEWAY,
http_client.SERVICE_UNAVAILABLE,
http_client.NOT_FOUND,
)
responses = [_make_response(status_code) for status_code in status_codes]
func = mock.Mock(side_effect=responses, spec=[])
retry_strategy = common.RetryStrategy()
ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
assert ret_val == responses[-1]
assert status_codes[-1] not in _helpers.RETRYABLE
assert func.call_count == 4
assert func.mock_calls == [mock.call()] * 4
assert randint_mock.call_count == 3
assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
assert sleep_mock.call_count == 3
sleep_mock.assert_any_call(1.125)
sleep_mock.assert_any_call(2.625)
sleep_mock.assert_any_call(4.375)
def test_success_with_retry_connection_error(self, randint_mock,
sleep_mock):
randint_mock.side_effect = [125, 625, 375]
response = _make_response(http_client.NOT_FOUND)
responses = [
requests.exceptions.ConnectionError,
requests.exceptions.ConnectionError,
requests.exceptions.ConnectionError,
response,
]
func = mock.Mock(side_effect=responses, spec=[])
retry_strategy = common.RetryStrategy()
ret_val = _helpers.wait_and_retry(func, _get_status_code,
retry_strategy)
assert ret_val == responses[-1]
assert func.call_count == 4
assert func.mock_calls == [mock.call()] * 4
assert randint_mock.call_count == 3
assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
assert sleep_mock.call_count == 3
sleep_mock.assert_any_call(1.125)
sleep_mock.assert_any_call(2.625)
sleep_mock.assert_any_call(4.375)
def test_success_with_retry_custom_delay(self, randint_mock, sleep_mock):
randint_mock.side_effect = [125, 625, 375]
status_codes = (
http_client.INTERNAL_SERVER_ERROR,
http_client.BAD_GATEWAY,
http_client.SERVICE_UNAVAILABLE,
http_client.NOT_FOUND,
)
responses = [
_make_response(status_code) for status_code in status_codes
]
func = mock.Mock(side_effect=responses, spec=[])
retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4)
ret_val = _helpers.wait_and_retry(func, _get_status_code,
retry_strategy)
assert ret_val == responses[-1]
assert status_codes[-1] not in common.RETRYABLE
assert func.call_count == 4
assert func.mock_calls == [mock.call()] * 4
assert randint_mock.call_count == 3
assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
assert sleep_mock.call_count == 3
sleep_mock.assert_any_call(3.125) # initial delay 3 + jitter 0.125
sleep_mock.assert_any_call(
12.625) # previous delay 3 * multiplier 4 + jitter 0.625
sleep_mock.assert_any_call(
48.375) # previous delay 12 * multiplier 4 + jitter 0.375
def http_request(transport, method, url, data=None, headers=None,
retry_strategy=_DEFAULT_RETRY_STRATEGY, **transport_kwargs):
"""Make an HTTP request.
Args:
transport (~requests.Session): A ``requests`` object which can make
authenticated requests via a ``request()`` method. This method
must accept an HTTP method, an upload URL, a ``data`` keyword
argument and a ``headers`` keyword argument.
method (str): The HTTP method for the request.
url (str): The URL for the request.
data (Optional[bytes]): The body of the request.
headers (Mapping[str, str]): The headers for the request (``transport``
may also add additional headers).
retry_strategy (~google.resumable_media.common.RetryStrategy): The
strategy to use if the request fails and must be retried.
transport_kwargs (Dict[str, str]): Extra keyword arguments to be
passed along to ``transport.request``.
Returns:
~requests.Response: The return value of ``transport.request()``.
"""
func = functools.partial(
transport.request, method, url, data=data, headers=headers,
**transport_kwargs)
return _helpers.wait_and_retry(
func, RequestsMixin._get_status_code, retry_strategy)
def consume(
self,
transport,
timeout=(
_request_helpers._DEFAULT_CONNECT_TIMEOUT,
_request_helpers._DEFAULT_READ_TIMEOUT,
),
):
"""Consume the resource to be downloaded.
If a ``stream`` is attached to this download, then the downloaded
resource will be written to the stream.
Args:
transport (~requests.Session): A ``requests`` object which can
make authenticated requests.
timeout (Optional[Union[float, Tuple[float, float]]]):
The number of seconds to wait for the server response.
Depending on the retry strategy, a request may be repeated
several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
Returns:
~requests.Response: The HTTP response returned by ``transport``.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
ValueError: If the current :class:`Download` has already
finished.
"""
method, url, payload, headers = self._prepare_request()
# Wrap the request business logic in a function to be retried.
def retriable_request():
# NOTE: We assume "payload is None" but pass it along anyway.
result = transport.request(
method,
url,
data=payload,
headers=headers,
stream=True,
timeout=timeout,
)
self._process_response(result)
if self._stream is not None:
self._write_to_stream(result)
return result
return _helpers.wait_and_retry(retriable_request,
self._get_status_code,
self._retry_strategy)
def test_success_no_retry(self):
truthy = http_client.OK
assert truthy not in _helpers.RETRYABLE
response = _make_response(truthy)
func = mock.Mock(return_value=response, spec=[])
retry_strategy = common.RetryStrategy()
ret_val = _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
assert ret_val is response
func.assert_called_once_with()
def test_connection_import_error_failure(self, randint_mock, sleep_mock):
randint_mock.side_effect = [125, 625, 375]
response = _make_response(http_client.NOT_FOUND)
responses = [
requests.exceptions.ConnectionError,
requests.exceptions.ConnectionError,
requests.exceptions.ConnectionError,
response,
]
with mock.patch(
"google.resumable_media._helpers._get_connection_error_classes",
side_effect=ImportError,
):
with pytest.raises(requests.exceptions.ConnectionError):
func = mock.Mock(side_effect=responses, spec=[])
retry_strategy = common.RetryStrategy()
_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
def transmit(
self,
transport,
data,
metadata,
content_type,
timeout=(
_request_helpers._DEFAULT_CONNECT_TIMEOUT,
_request_helpers._DEFAULT_READ_TIMEOUT,
),
):
"""Transmit the resource to be uploaded.
Args:
transport (~requests.Session): A ``requests`` object which can
make authenticated requests.
data (bytes): The resource content to be uploaded.
metadata (Mapping[str, str]): The resource metadata, such as an
ACL list.
content_type (str): The content type of the resource, e.g. a JPEG
image has content type ``image/jpeg``.
timeout (Optional[Union[float, Tuple[float, float]]]):
The number of seconds to wait for the server response.
Depending on the retry strategy, a request may be repeated
several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
Returns:
~requests.Response: The HTTP response returned by ``transport``.
"""
method, url, payload, headers = self._prepare_request(
data, metadata, content_type)
# Wrap the request business logic in a function to be retried.
def retriable_request():
result = transport.request(method,
url,
data=payload,
headers=headers,
timeout=timeout)
self._process_response(result)
return result
return _helpers.wait_and_retry(retriable_request,
self._get_status_code,
self._retry_strategy)
def consume_next_chunk(
self,
transport,
timeout=(
_request_helpers._DEFAULT_CONNECT_TIMEOUT,
_request_helpers._DEFAULT_READ_TIMEOUT,
),
):
"""Consume the next chunk of the resource to be downloaded.
Args:
transport (~requests.Session): A ``requests`` object which can
make authenticated requests.
timeout (Optional[Union[float, Tuple[float, float]]]):
The number of seconds to wait for the server response.
Depending on the retry strategy, a request may be repeated
several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
Returns:
~requests.Response: The HTTP response returned by ``transport``.
Raises:
ValueError: If the current download has finished.
"""
method, url, payload, headers = self._prepare_request()
# Wrap the request business logic in a function to be retried.
def retriable_request():
# NOTE: We assume "payload is None" but pass it along anyway.
result = transport.request(
method,
url,
data=payload,
headers=headers,
stream=True,
timeout=timeout,
)
self._process_response(result)
return result
return _helpers.wait_and_retry(retriable_request,
self._get_status_code,
self._retry_strategy)
def recover(self, transport):
"""Recover from a failure.
This method should be used when a :class:`ResumableUpload` is in an
:attr:`~ResumableUpload.invalid` state due to a request failure.
This will verify the progress with the server and make sure the
current upload is in a valid state before :meth:`transmit_next_chunk`
can be used again.
Args:
transport (~requests.Session): A ``requests`` object which can
make authenticated requests.
Returns:
~requests.Response: The HTTP response returned by ``transport``.
"""
timeout = (
_request_helpers._DEFAULT_CONNECT_TIMEOUT,
_request_helpers._DEFAULT_READ_TIMEOUT,
)
method, url, payload, headers = self._prepare_recover_request()
# NOTE: We assume "payload is None" but pass it along anyway.
# Wrap the request business logic in a function to be retried.
def retriable_request():
result = transport.request(method,
url,
data=payload,
headers=headers,
timeout=timeout)
self._process_recover_response(result)
return result
return _helpers.wait_and_retry(retriable_request,
self._get_status_code,
self._retry_strategy)
def test_retry_exceeds_max_cumulative(self, randint_mock, sleep_mock):
randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
status_codes = (
http_client.SERVICE_UNAVAILABLE,
http_client.GATEWAY_TIMEOUT,
common.TOO_MANY_REQUESTS,
http_client.INTERNAL_SERVER_ERROR,
http_client.SERVICE_UNAVAILABLE,
http_client.BAD_GATEWAY,
http_client.GATEWAY_TIMEOUT,
common.TOO_MANY_REQUESTS,
)
responses = [
_make_response(status_code) for status_code in status_codes
]
func = mock.Mock(side_effect=responses, spec=[])
retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
ret_val = _helpers.wait_and_retry(func, _get_status_code,
retry_strategy)
assert ret_val == responses[-1]
assert status_codes[-1] in _helpers.RETRYABLE
assert func.call_count == 8
assert func.mock_calls == [mock.call()] * 8
assert randint_mock.call_count == 7
assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
assert sleep_mock.call_count == 7
sleep_mock.assert_any_call(1.875)
sleep_mock.assert_any_call(2.0)
sleep_mock.assert_any_call(4.375)
sleep_mock.assert_any_call(8.5)
sleep_mock.assert_any_call(16.5)
sleep_mock.assert_any_call(32.25)
sleep_mock.assert_any_call(64.125)
def transmit_next_chunk(
self,
transport,
timeout=(
_request_helpers._DEFAULT_CONNECT_TIMEOUT,
_request_helpers._DEFAULT_READ_TIMEOUT,
),
):
"""Transmit the next chunk of the resource to be uploaded.
If the current upload was initiated with ``stream_final=False``,
this method will dynamically determine if the upload has completed.
The upload will be considered complete if the stream produces
fewer than :attr:`chunk_size` bytes when a chunk is read from it.
In the case of failure, an exception is thrown that preserves the
failed response:
.. testsetup:: bad-response
import io
import mock
import requests
import http.client
from google import resumable_media
import google.resumable_media.requests.upload as upload_mod
transport = mock.Mock(spec=['request'])
fake_response = requests.Response()
fake_response.status_code = int(http.client.BAD_REQUEST)
transport.request.return_value = fake_response
upload_url = 'http://test.invalid'
upload = upload_mod.ResumableUpload(
upload_url, resumable_media.UPLOAD_CHUNK_SIZE)
# Fake that the upload has been initiate()-d
data = b'data is here'
upload._stream = io.BytesIO(data)
upload._total_bytes = len(data)
upload._resumable_url = 'http://test.invalid?upload_id=nope'
.. doctest:: bad-response
:options: +NORMALIZE_WHITESPACE
>>> error = None
>>> try:
... upload.transmit_next_chunk(transport)
... except resumable_media.InvalidResponse as caught_exc:
... error = caught_exc
...
>>> error
InvalidResponse('Request failed with status code', 400,
'Expected one of', <HTTPStatus.OK: 200>, <HTTPStatus.PERMANENT_REDIRECT: 308>)
>>> error.response
<Response [400]>
Args:
transport (~requests.Session): A ``requests`` object which can
make authenticated requests.
timeout (Optional[Union[float, Tuple[float, float]]]):
The number of seconds to wait for the server response.
Depending on the retry strategy, a request may be repeated
several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
Returns:
~requests.Response: The HTTP response returned by ``transport``.
Raises:
~google.resumable_media.common.InvalidResponse: If the status
code is not 200 or http.client.PERMANENT_REDIRECT.
~google.resumable_media.common.DataCorruption: If this is the final
chunk, a checksum validation was requested, and the checksum
does not match or is not available.
"""
method, url, payload, headers = self._prepare_request()
# Wrap the request business logic in a function to be retried.
def retriable_request():
result = transport.request(method,
url,
data=payload,
headers=headers,
timeout=timeout)
self._process_response(result, len(payload))
return result
return _helpers.wait_and_retry(retriable_request,
self._get_status_code,
self._retry_strategy)
def initiate(
self,
transport,
stream,
metadata,
content_type,
total_bytes=None,
stream_final=True,
timeout=(
_request_helpers._DEFAULT_CONNECT_TIMEOUT,
_request_helpers._DEFAULT_READ_TIMEOUT,
),
):
"""Initiate a resumable upload.
By default, this method assumes your ``stream`` is in a "final"
state ready to transmit. However, ``stream_final=False`` can be used
to indicate that the size of the resource is not known. This can happen
if bytes are being dynamically fed into ``stream``, e.g. if the stream
is attached to application logs.
If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be
read from the stream every time :meth:`transmit_next_chunk` is called.
If one of those reads produces strictly fewer bites than the chunk
size, the upload will be concluded.
Args:
transport (~requests.Session): A ``requests`` object which can
make authenticated requests.
stream (IO[bytes]): The stream (i.e. file-like object) that will
be uploaded. The stream **must** be at the beginning (i.e.
``stream.tell() == 0``).
metadata (Mapping[str, str]): The resource metadata, such as an
ACL list.
content_type (str): The content type of the resource, e.g. a JPEG
image has content type ``image/jpeg``.
total_bytes (Optional[int]): The total number of bytes to be
uploaded. If specified, the upload size **will not** be
determined from the stream (even if ``stream_final=True``).
stream_final (Optional[bool]): Indicates if the ``stream`` is
"final" (i.e. no more bytes will be added to it). In this case
we determine the upload size from the size of the stream. If
``total_bytes`` is passed, this argument will be ignored.
timeout (Optional[Union[float, Tuple[float, float]]]):
The number of seconds to wait for the server response.
Depending on the retry strategy, a request may be repeated
several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
Returns:
~requests.Response: The HTTP response returned by ``transport``.
"""
method, url, payload, headers = self._prepare_initiate_request(
stream,
metadata,
content_type,
total_bytes=total_bytes,
stream_final=stream_final,
)
# Wrap the request business logic in a function to be retried.
def retriable_request():
result = transport.request(method,
url,
data=payload,
headers=headers,
timeout=timeout)
self._process_initiate_response(result)
return result
return _helpers.wait_and_retry(retriable_request,
self._get_status_code,
self._retry_strategy)
