async def _validate_checksum(self, response):
"""Check the computed checksum, if any, against the response headers.
Args:
response (object): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the checksum
computed locally and the checksum reported by the remote host do
not match.
"""
if self._checksum_type is None:
return
metadata_key = sync_helpers._get_metadata_key(self._checksum_type)
metadata = await response.json()
remote_checksum = metadata.get(metadata_key)
if remote_checksum is None:
raise common.InvalidResponse(
response,
_UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(
metadata_key),
self._get_headers(response),
)
local_checksum = sync_helpers.prepare_checksum_digest(
self._checksum_object.digest())
if local_checksum != remote_checksum:
raise common.DataCorruption(
response,
_UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
self._checksum_type.upper(), local_checksum,
remote_checksum),
)
def _write_to_stream(self, response):
"""Write response body to a write-able stream.
.. note:
This method assumes that the ``_stream`` attribute is set on the
current download.
Args:
response (~requests.Response): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
"""
# `_get_expected_checksum()` may return None even if a checksum was
# requested, in which case it will emit an info log _MISSING_CHECKSUM.
# If an invalid checksum type is specified, this will raise ValueError.
expected_checksum, checksum_object = _get_expected_checksum(
response, self._get_headers, self.media_url, checksum_type=self.checksum
)
with response:
# NOTE: In order to handle compressed streams gracefully, we try
# to insert our checksum object into the decompression stream. If
# the stream is indeed compressed, this will delegate the checksum
# object to the decoder and return a _DoNothingHash here.
local_checksum_object = _add_decoder(response.raw, checksum_object)
body_iter = response.iter_content(
chunk_size=_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
)
for chunk in body_iter:
self._stream.write(chunk)
local_checksum_object.update(chunk)
if expected_checksum is None:
return
else:
actual_checksum = base64.b64encode(checksum_object.digest())
# NOTE: ``b64encode`` returns ``bytes``, but ``expected_checksum``
# came from a header, so it will be ``str``.
actual_checksum = actual_checksum.decode(u"utf-8")
if actual_checksum != expected_checksum:
msg = _CHECKSUM_MISMATCH.format(
self.media_url,
expected_checksum,
actual_checksum,
checksum_type=self.checksum.upper(),
)
raise common.DataCorruption(response, msg)
def _write_to_stream(self, response):
"""Write response body to a write-able stream.
.. note:
This method assumes that the ``_stream`` attribute is set on the
current download.
Args:
response (~requests.Response): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
"""
# `_get_expected_checksum()` may return None even if a checksum was
# requested, in which case it will emit an info log _MISSING_CHECKSUM.
# If an invalid checksum type is specified, this will raise ValueError.
expected_checksum, checksum_object = _get_expected_checksum(
response, self._get_headers, self.media_url, checksum_type=self.checksum
)
with response:
body_iter = response.raw.stream(
_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
)
for chunk in body_iter:
self._stream.write(chunk)
checksum_object.update(chunk)
response._content_consumed = True
if expected_checksum is None:
return
else:
actual_checksum = base64.b64encode(checksum_object.digest())
# NOTE: ``b64encode`` returns ``bytes``, but ``expected_checksum``
# came from a header, so it will be ``str``.
actual_checksum = actual_checksum.decode(u"utf-8")
if actual_checksum != expected_checksum:
msg = _CHECKSUM_MISMATCH.format(
self.media_url,
expected_checksum,
actual_checksum,
checksum_type=self.checksum.upper(),
)
raise common.DataCorruption(response, msg)
async def _write_to_stream(self, response):
"""Write response body to a write-able stream.
.. note:
This method assumes that the ``_stream`` attribute is set on the
current download.
Args:
response (~requests.Response): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
"""
# `_get_expected_checksum()` may return None even if a checksum was
# requested, in which case it will emit an info log _MISSING_CHECKSUM.
# If an invalid checksum type is specified, this will raise ValueError.
expected_checksum, checksum_object = sync_helpers._get_expected_checksum(
response,
self._get_headers,
self.media_url,
checksum_type=self.checksum)
local_checksum_object = _add_decoder(response, checksum_object)
async for chunk in response.content.iter_chunked(
_request_helpers._SINGLE_GET_CHUNK_SIZE):
self._stream.write(chunk)
local_checksum_object.update(chunk)
if expected_checksum is None:
return
else:
actual_checksum = sync_helpers.prepare_checksum_digest(
checksum_object.digest())
if actual_checksum != expected_checksum:
msg = _CHECKSUM_MISMATCH.format(
self.media_url,
expected_checksum,
actual_checksum,
checksum_type=self.checksum.upper(),
)
raise common.DataCorruption(response, msg)
def _write_to_stream(self, response):
"""Write response body to a write-able stream.
.. note:
This method assumes that the ``_stream`` attribute is set on the
current download.
Args:
response (~requests.Response): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
"""
expected_md5_hash = _get_expected_md5(response, self._get_headers,
self.media_url)
if expected_md5_hash is None:
md5_hash = _DoNothingHash()
else:
md5_hash = hashlib.md5()
with response:
# NOTE: This might "donate" ``md5_hash`` to the decoder and replace
# it with a ``_DoNothingHash``.
local_hash = _add_decoder(response.raw, md5_hash)
body_iter = response.iter_content(
chunk_size=_helpers._SINGLE_GET_CHUNK_SIZE,
decode_unicode=False)
for chunk in body_iter:
self._stream.write(chunk)
local_hash.update(chunk)
if expected_md5_hash is None:
return
actual_md5_hash = base64.b64encode(md5_hash.digest())
# NOTE: ``b64encode`` returns ``bytes``, but ``expected_md5_hash``
# came from a header, so it will be ``str``.
actual_md5_hash = actual_md5_hash.decode(u"utf-8")
if actual_md5_hash != expected_md5_hash:
msg = _CHECKSUM_MISMATCH.format(self.media_url, expected_md5_hash,
actual_md5_hash)
raise common.DataCorruption(response, msg)