async def _validate_checksum(self, response): """Check the computed checksum, if any, against the response headers. Args: response (object): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the checksum computed locally and the checksum reported by the remote host do not match. """ if self._checksum_type is None: return metadata_key = sync_helpers._get_metadata_key(self._checksum_type) metadata = await response.json() remote_checksum = metadata.get(metadata_key) if remote_checksum is None: raise common.InvalidResponse( response, _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format( metadata_key), self._get_headers(response), ) local_checksum = sync_helpers.prepare_checksum_digest( self._checksum_object.digest()) if local_checksum != remote_checksum: raise common.DataCorruption( response, _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( self._checksum_type.upper(), local_checksum, remote_checksum), )
def _write_to_stream(self, response): """Write response body to a write-able stream. .. note: This method assumes that the ``_stream`` attribute is set on the current download. Args: response (~requests.Response): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ # `_get_expected_checksum()` may return None even if a checksum was # requested, in which case it will emit an info log _MISSING_CHECKSUM. # If an invalid checksum type is specified, this will raise ValueError. expected_checksum, checksum_object = _get_expected_checksum( response, self._get_headers, self.media_url, checksum_type=self.checksum ) with response: # NOTE: In order to handle compressed streams gracefully, we try # to insert our checksum object into the decompression stream. If # the stream is indeed compressed, this will delegate the checksum # object to the decoder and return a _DoNothingHash here. local_checksum_object = _add_decoder(response.raw, checksum_object) body_iter = response.iter_content( chunk_size=_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False ) for chunk in body_iter: self._stream.write(chunk) local_checksum_object.update(chunk) if expected_checksum is None: return else: actual_checksum = base64.b64encode(checksum_object.digest()) # NOTE: ``b64encode`` returns ``bytes``, but ``expected_checksum`` # came from a header, so it will be ``str``. actual_checksum = actual_checksum.decode(u"utf-8") if actual_checksum != expected_checksum: msg = _CHECKSUM_MISMATCH.format( self.media_url, expected_checksum, actual_checksum, checksum_type=self.checksum.upper(), ) raise common.DataCorruption(response, msg)
def _write_to_stream(self, response): """Write response body to a write-able stream. .. note: This method assumes that the ``_stream`` attribute is set on the current download. Args: response (~requests.Response): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ # `_get_expected_checksum()` may return None even if a checksum was # requested, in which case it will emit an info log _MISSING_CHECKSUM. # If an invalid checksum type is specified, this will raise ValueError. expected_checksum, checksum_object = _get_expected_checksum( response, self._get_headers, self.media_url, checksum_type=self.checksum ) with response: body_iter = response.raw.stream( _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False ) for chunk in body_iter: self._stream.write(chunk) checksum_object.update(chunk) response._content_consumed = True if expected_checksum is None: return else: actual_checksum = base64.b64encode(checksum_object.digest()) # NOTE: ``b64encode`` returns ``bytes``, but ``expected_checksum`` # came from a header, so it will be ``str``. actual_checksum = actual_checksum.decode(u"utf-8") if actual_checksum != expected_checksum: msg = _CHECKSUM_MISMATCH.format( self.media_url, expected_checksum, actual_checksum, checksum_type=self.checksum.upper(), ) raise common.DataCorruption(response, msg)
async def _write_to_stream(self, response): """Write response body to a write-able stream. .. note: This method assumes that the ``_stream`` attribute is set on the current download. Args: response (~requests.Response): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ # `_get_expected_checksum()` may return None even if a checksum was # requested, in which case it will emit an info log _MISSING_CHECKSUM. # If an invalid checksum type is specified, this will raise ValueError. expected_checksum, checksum_object = sync_helpers._get_expected_checksum( response, self._get_headers, self.media_url, checksum_type=self.checksum) local_checksum_object = _add_decoder(response, checksum_object) async for chunk in response.content.iter_chunked( _request_helpers._SINGLE_GET_CHUNK_SIZE): self._stream.write(chunk) local_checksum_object.update(chunk) if expected_checksum is None: return else: actual_checksum = sync_helpers.prepare_checksum_digest( checksum_object.digest()) if actual_checksum != expected_checksum: msg = _CHECKSUM_MISMATCH.format( self.media_url, expected_checksum, actual_checksum, checksum_type=self.checksum.upper(), ) raise common.DataCorruption(response, msg)
def _write_to_stream(self, response): """Write response body to a write-able stream. .. note: This method assumes that the ``_stream`` attribute is set on the current download. Args: response (~requests.Response): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ expected_md5_hash = _get_expected_md5(response, self._get_headers, self.media_url) if expected_md5_hash is None: md5_hash = _DoNothingHash() else: md5_hash = hashlib.md5() with response: # NOTE: This might "donate" ``md5_hash`` to the decoder and replace # it with a ``_DoNothingHash``. local_hash = _add_decoder(response.raw, md5_hash) body_iter = response.iter_content( chunk_size=_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False) for chunk in body_iter: self._stream.write(chunk) local_hash.update(chunk) if expected_md5_hash is None: return actual_md5_hash = base64.b64encode(md5_hash.digest()) # NOTE: ``b64encode`` returns ``bytes``, but ``expected_md5_hash`` # came from a header, so it will be ``str``. actual_md5_hash = actual_md5_hash.decode(u"utf-8") if actual_md5_hash != expected_md5_hash: msg = _CHECKSUM_MISMATCH.format(self.media_url, expected_md5_hash, actual_md5_hash) raise common.DataCorruption(response, msg)