def _failure_helper(self, **kwargs):
response = mock.Mock(headers={}, spec=["headers"])
name = u"any-name"
with pytest.raises(common.InvalidResponse) as exc_info:
_helpers.header_required(response, name, _get_headers, **kwargs)
error = exc_info.value
assert error.response is response
assert len(error.args) == 2
assert error.args[1] == name
def _check_for_zero_content_range(response, get_status_code, get_headers):
""" Validate if response status code is 416 and content range is zero.
This is the special case for handling zero bytes files.
Args:
response (object): An HTTP response object.
get_status_code (Callable[Any, int]): Helper to get a status code
from a response.
get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
from an HTTP response.
Returns:
bool: True if content range total bytes is zero, false otherwise.
"""
if get_status_code(response) == http_client.REQUESTED_RANGE_NOT_SATISFIABLE:
content_range = _helpers.header_required(
response,
_helpers.CONTENT_RANGE_HEADER,
get_headers,
callback=_helpers.do_nothing,
)
if content_range == _ZERO_CONTENT_RANGE_HEADER:
return True
return False
def get_range_info(response, get_headers, callback=_helpers.do_nothing):
"""Get the start, end and total bytes from a content range header.
Args:
response (object): An HTTP response object.
get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
from an HTTP response.
callback (Optional[Callable]): A callback that takes no arguments,
to be executed when an exception is being raised.
Returns:
Tuple[int, int, int]: The start byte, end byte and total bytes.
Raises:
~google.resumable_media.common.InvalidResponse: If the
``Content-Range`` header is not of the form
``bytes {start}-{end}/{total}``.
"""
content_range = _helpers.header_required(response,
_helpers.CONTENT_RANGE_HEADER,
get_headers,
callback=callback)
match = _CONTENT_RANGE_RE.match(content_range)
if match is None:
callback()
raise common.InvalidResponse(
response, u'Unexpected content-range header', content_range,
u'Expected to be of the form "bytes {start}-{end}/{total}"')
return (int(match.group(u'start_byte')), int(match.group(u'end_byte')),
int(match.group(u'total_bytes')))
def _success_helper(self, **kwargs):
name = u"some-header"
value = u"The Right Hand Side"
headers = {name: value, u"other-name": u"other-value"}
response = mock.Mock(headers=headers, spec=["headers"])
result = _helpers.header_required(response, name, _get_headers, **kwargs)
assert result == value
def _process_initiate_response(self, response):
"""Process the response from an HTTP request that initiated upload.
This is everything that must be done after a request that doesn't
require network I/O (or other I/O). This is based on the `sans-I/O`_
philosophy.
This method takes the URL from the ``Location`` header and stores it
for future use. Within that URL, we assume the ``upload_id`` query
parameter has been included, but we do not check.
Args:
response (object): The HTTP response object (need headers).
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
_helpers.require_status_code(
response,
(http.client.OK, http.client.CREATED),
self._get_status_code,
callback=self._make_invalid,
)
self._resumable_url = _helpers.header_required(
response, "location", self._get_headers
)
def get_range_info(response, get_headers, callback=_helpers.do_nothing):
"""Get the start, end and total bytes from a content range header.
Args:
response (object): An HTTP response object.
get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
from an HTTP response.
callback (Optional[Callable]): A callback that takes no arguments,
to be executed when an exception is being raised.
Returns:
Tuple[int, int, int]: The start byte, end byte and total bytes.
Raises:
~google.resumable_media.common.InvalidResponse: If the
``Content-Range`` header is not of the form
``bytes {start}-{end}/{total}``.
"""
content_range = _helpers.header_required(
response, _helpers.CONTENT_RANGE_HEADER,
get_headers, callback=callback)
match = _CONTENT_RANGE_RE.match(content_range)
if match is None:
callback()
raise common.InvalidResponse(
response, u'Unexpected content-range header', content_range,
u'Expected to be of the form "bytes {start}-{end}/{total}"')
return (
int(match.group(u'start_byte')),
int(match.group(u'end_byte')),
int(match.group(u'total_bytes'))
)
def _process_response(self, response):
"""Process the response from an HTTP request.
This is everything that must be done after a request that doesn't
require network I/O (or other I/O). This is based on the `sans-I/O`_
philosophy.
Updates the current state after consuming a chunk. First,
increments ``bytes_downloaded`` by the number of bytes in the
``content-length`` header.
If ``total_bytes`` is already set, this assumes (but does not check)
that we already have the correct value and doesn't bother to check
that it agrees with the headers.
We expect the **total** length to be in the ``content-range`` header,
but this header is only present on requests which sent the ``range``
header. This response header should be of the form
``bytes {start}-{end}/{total}`` and ``{end} - {start} + 1``
should be the same as the ``Content-Length``.
Args:
response (object): The HTTP response object (need headers).
Raises:
~google.resumable_media.exceptions.InvalidResponse: If the number
of bytes in the body doesn't match the content length header.
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
# Verify the response before updating the current instance.
_helpers.require_status_code(response,
_ACCEPTABLE_STATUS_CODES,
callback=self._make_invalid)
content_length = _helpers.header_required(response,
u'content-length',
callback=self._make_invalid)
num_bytes = int(content_length)
_, end_byte, total_bytes = _get_range_info(response,
callback=self._make_invalid)
response_body = _helpers.get_body(response)
if len(response_body) != num_bytes:
self._make_invalid()
raise exceptions.InvalidResponse(
response, u'Response is different size than content-length',
u'Expected', num_bytes, u'Received', len(response_body))
# First update ``bytes_downloaded``.
self._bytes_downloaded += num_bytes
# If the end byte is past ``end`` or ``total_bytes - 1`` we are done.
if self.end is not None and end_byte >= self.end:
self._finished = True
elif end_byte >= total_bytes - 1:
self._finished = True
# NOTE: We only use ``total_bytes`` if not already known.
if self.total_bytes is None:
self._total_bytes = total_bytes
# Write the response body to the stream.
self._stream.write(response_body)
def _success_helper(self, **kwargs):
name = u'some-header'
value = u'The Right Hand Side'
headers = {name: value, u'other-name': u'other-value'}
response = mock.Mock(headers=headers, spec=[u'headers'])
result = _helpers.header_required(response, name, _get_headers,
**kwargs)
assert result == value
def _process_response(self, response, bytes_sent):
"""Process the response from an HTTP request.
This is everything that must be done after a request that doesn't
require network I/O (or other I/O). This is based on the `sans-I/O`_
philosophy.
Args:
response (object): The HTTP response object.
bytes_sent (int): The number of bytes sent in the request that
``response`` was returned for.
Raises:
~google.resumable_media.common.InvalidResponse: If the status
code is 308 and the ``range`` header is not of the form
``bytes 0-{end}``.
~google.resumable_media.common.InvalidResponse: If the status
code is not 200 or 308.
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
status_code = _helpers.require_status_code(
response,
(http_client.OK, resumable_media.PERMANENT_REDIRECT),
self._get_status_code,
callback=self._make_invalid,
)
if status_code == http_client.OK:
# NOTE: We use the "local" information of ``bytes_sent`` to update
# ``bytes_uploaded``, but do not verify this against other
# state. However, there may be some other information:
#
# * a ``size`` key in JSON response body
# * the ``total_bytes`` attribute (if set)
# * ``stream.tell()`` (relying on fact that ``initiate()``
# requires stream to be at the beginning)
self._bytes_uploaded = self._bytes_uploaded + bytes_sent
# Tombstone the current upload so it cannot be used again.
self._finished = True
# Validate the checksum. This can raise an exception on failure.
self._validate_checksum(response)
else:
bytes_range = _helpers.header_required(
response,
_helpers.RANGE_HEADER,
self._get_headers,
callback=self._make_invalid,
)
match = _BYTES_RANGE_RE.match(bytes_range)
if match is None:
self._make_invalid()
raise common.InvalidResponse(
response,
u'Unexpected "range" header',
bytes_range,
u'Expected to be of the form "bytes=0-{end}"',
)
self._bytes_uploaded = int(match.group(u"end_byte")) + 1
def _process_response(self, response):
"""Process the response from an HTTP request.
This is everything that must be done after a request that doesn't
require network I/O (or other I/O). This is based on the `sans-I/O`_
philosophy.
Args:
response (object): The HTTP response object.
Raises:
~google.resumable_media.exceptions.InvalidResponse: If the status
code is 308 and the ``range`` header is not of the form
``bytes 0-{end}``.
~google.resumable_media.exceptions.InvalidResponse: If the status
code is not 200 or 308.
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
status_code = _helpers.require_status_code(
response, (http_client.OK, resumable_media.PERMANENT_REDIRECT),
self._get_status_code,
callback=self._make_invalid)
if status_code == http_client.OK:
json_response = json.loads(self._get_body(response))
self._bytes_uploaded = int(json_response[u'size'])
# Tombstone the current upload so it cannot be used again.
self._finished = True
else:
bytes_range = _helpers.header_required(response,
_helpers.RANGE_HEADER,
self._get_headers,
callback=self._make_invalid)
match = _BYTES_RANGE_RE.match(bytes_range)
if match is None:
self._make_invalid()
raise exceptions.InvalidResponse(
response, u'Unexpected "range" header', bytes_range,
u'Expected to be of the form "bytes=0-{end}"')
self._bytes_uploaded = int(match.group(u'end_byte')) + 1
def _process_response(self, response):
"""Process the response from an HTTP request.
This is everything that must be done after a request that doesn't
require network I/O. This is based on the `sans-I/O`_ philosophy.
For the time being, this **does require** some form of I/O to write
a chunk to ``stream``. However, this will (almost) certainly not be
network I/O.
Updates the current state after consuming a chunk. First,
increments ``bytes_downloaded`` by the number of bytes in the
``content-length`` header.
If ``total_bytes`` is already set, this assumes (but does not check)
that we already have the correct value and doesn't bother to check
that it agrees with the headers.
We expect the **total** length to be in the ``content-range`` header,
but this header is only present on requests which sent the ``range``
header. This response header should be of the form
``bytes {start}-{end}/{total}`` and ``{end} - {start} + 1``
should be the same as the ``Content-Length``.
Args:
response (object): The HTTP response object (need headers).
Raises:
~google.resumable_media.common.InvalidResponse: If the number
of bytes in the body doesn't match the content length header.
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
# Verify the response before updating the current instance.
if _check_for_zero_content_range(
response, self._get_status_code, self._get_headers
):
self._finished = True
return
_helpers.require_status_code(
response,
_ACCEPTABLE_STATUS_CODES,
self._get_status_code,
callback=self._make_invalid,
)
headers = self._get_headers(response)
response_body = self._get_body(response)
start_byte, end_byte, total_bytes = get_range_info(
response, self._get_headers, callback=self._make_invalid
)
transfer_encoding = headers.get(u"transfer-encoding")
if transfer_encoding is None:
content_length = _helpers.header_required(
response,
u"content-length",
self._get_headers,
callback=self._make_invalid,
)
num_bytes = int(content_length)
if len(response_body) != num_bytes:
self._make_invalid()
raise common.InvalidResponse(
response,
u"Response is different size than content-length",
u"Expected",
num_bytes,
u"Received",
len(response_body),
)
else:
# 'content-length' header not allowed with chunked encoding.
num_bytes = end_byte - start_byte + 1
# First update ``bytes_downloaded``.
self._bytes_downloaded += num_bytes
# If the end byte is past ``end`` or ``total_bytes - 1`` we are done.
if self.end is not None and end_byte >= self.end:
self._finished = True
elif end_byte >= total_bytes - 1:
self._finished = True
# NOTE: We only use ``total_bytes`` if not already known.
if self.total_bytes is None:
self._total_bytes = total_bytes
# Write the response body to the stream.
self._stream.write(response_body)
def _process_response(self, response):
"""Process the response from an HTTP request.
This is everything that must be done after a request that doesn't
require network I/O. This is based on the `sans-I/O`_ philosophy.
For the time being, this **does require** some form of I/O to write
a chunk to ``stream``. However, this will (almost) certainly not be
network I/O.
Updates the current state after consuming a chunk. First,
increments ``bytes_downloaded`` by the number of bytes in the
``content-length`` header.
If ``total_bytes`` is already set, this assumes (but does not check)
that we already have the correct value and doesn't bother to check
that it agrees with the headers.
We expect the **total** length to be in the ``content-range`` header,
but this header is only present on requests which sent the ``range``
header. This response header should be of the form
``bytes {start}-{end}/{total}`` and ``{end} - {start} + 1``
should be the same as the ``Content-Length``.
Args:
response (object): The HTTP response object (need headers).
Raises:
~google.resumable_media.common.InvalidResponse: If the number
of bytes in the body doesn't match the content length header.
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
# Verify the response before updating the current instance.
_helpers.require_status_code(
response, _ACCEPTABLE_STATUS_CODES,
self._get_status_code, callback=self._make_invalid)
content_length = _helpers.header_required(
response, u'content-length', self._get_headers,
callback=self._make_invalid)
num_bytes = int(content_length)
_, end_byte, total_bytes = get_range_info(
response, self._get_headers, callback=self._make_invalid)
response_body = self._get_body(response)
if len(response_body) != num_bytes:
self._make_invalid()
raise common.InvalidResponse(
response, u'Response is different size than content-length',
u'Expected', num_bytes, u'Received', len(response_body))
# First update ``bytes_downloaded``.
self._bytes_downloaded += num_bytes
# If the end byte is past ``end`` or ``total_bytes - 1`` we are done.
if self.end is not None and end_byte >= self.end:
self._finished = True
elif end_byte >= total_bytes - 1:
self._finished = True
# NOTE: We only use ``total_bytes`` if not already known.
if self.total_bytes is None:
self._total_bytes = total_bytes
# Write the response body to the stream.
self._stream.write(response_body)