def test__update_checksum_rewind(self, checksum):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=checksum)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 8
checksum_checkpoint = upload._checksum_object.digest()
# Rewind to the beginning.
upload._stream.seek(0)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 8
assert upload._checksum_object.digest() == checksum_checkpoint
# Rewind but not to the beginning.
upload._stream.seek(4)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 12
# Continue to the end.
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, len(data), len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == len(data)
checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
checksum_digest = sync_helpers.prepare_checksum_digest(
upload._checksum_object.digest())
assert checksum_digest == checksums[checksum]
async def _validate_checksum(self, response):
"""Check the computed checksum, if any, against the response headers.
Args:
response (object): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the checksum
computed locally and the checksum reported by the remote host do
not match.
"""
if self._checksum_type is None:
return
metadata_key = sync_helpers._get_metadata_key(self._checksum_type)
metadata = await response.json()
remote_checksum = metadata.get(metadata_key)
if remote_checksum is None:
raise common.InvalidResponse(
response,
_UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(
metadata_key),
self._get_headers(response),
)
local_checksum = sync_helpers.prepare_checksum_digest(
self._checksum_object.digest())
if local_checksum != remote_checksum:
raise common.DataCorruption(
response,
_UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
self._checksum_type.upper(), local_checksum,
remote_checksum),
)
def test_multipart_upload_with_bad_checksum(authorized_transport, checksum,
bucket):
with open(ICO_FILE, u"rb") as file_obj:
actual_contents = file_obj.read()
blob_name = os.path.basename(ICO_FILE)
check_does_not_exist(authorized_transport, blob_name)
# Create the actual upload object.
upload_url = utils.MULTIPART_UPLOAD
upload = resumable_requests.MultipartUpload(upload_url, checksum=checksum)
# Transmit the resource.
metadata = {u"name": blob_name, u"metadata": {u"color": u"yellow"}}
fake_checksum_object = _helpers._get_checksum_object(checksum)
fake_checksum_object.update(b"bad data")
fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
fake_checksum_object.digest())
with mock.patch.object(_helpers,
"prepare_checksum_digest",
return_value=fake_prepared_checksum_digest):
with pytest.raises(common.InvalidResponse) as exc_info:
response = upload.transmit(authorized_transport, actual_contents,
metadata, ICO_CONTENT_TYPE)
response = exc_info.value.response
message = response.json()["error"]["message"]
# Attempt to verify that this is a checksum mismatch error.
assert checksum.upper() in message
assert fake_prepared_checksum_digest in message
# Make sure the upload is tombstoned.
check_tombstoned(upload, authorized_transport, actual_contents, metadata,
ICO_CONTENT_TYPE)
async def test_resumable_upload_with_bad_checksum(authorized_transport,
img_stream, bucket, cleanup,
checksum):
fake_checksum_object = _helpers._get_checksum_object(checksum)
fake_checksum_object.update(b"bad data")
fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
fake_checksum_object.digest())
with mock.patch.object(_helpers,
"prepare_checksum_digest",
return_value=fake_prepared_checksum_digest):
with pytest.raises(common.DataCorruption) as exc_info:
await _resumable_upload_helper(authorized_transport,
img_stream,
cleanup,
checksum=checksum)
expected_checksums = {
"md5": "1bsd83IYNug8hd+V1ING3Q==",
"crc32c": "YQGPxA=="
}
expected_message = (_async_resumable_media._upload.
_UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
checksum.upper(),
fake_prepared_checksum_digest,
expected_checksums[checksum],
))
assert exc_info.value.args[0] == expected_message
def _prepare_request(self, data, metadata, content_type):
"""Prepare the contents of an HTTP request.
This is everything that must be done before a request that doesn't
require network I/O (or other I/O). This is based on the `sans-I/O`_
philosophy.
.. note:
This method will be used only once, so ``headers`` will be
mutated by having a new key added to it.
Args:
data (bytes): The resource content to be uploaded.
metadata (Mapping[str, str]): The resource metadata, such as an
ACL list.
content_type (str): The content type of the resource, e.g. a JPEG
image has content type ``image/jpeg``.
Returns:
Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
* HTTP verb for the request (always POST)
* the URL for the request
* the body of the request
* headers for the request
Raises:
ValueError: If the current upload has already finished.
TypeError: If ``data`` isn't bytes.
.. _sans-I/O: https://sans-io.readthedocs.io/
"""
if self.finished:
raise ValueError("An upload can only be used once.")
if not isinstance(data, bytes):
raise TypeError("`data` must be bytes, received", type(data))
checksum_object = sync_helpers._get_checksum_object(
self._checksum_type)
if checksum_object is not None:
checksum_object.update(data)
actual_checksum = sync_helpers.prepare_checksum_digest(
checksum_object.digest())
metadata_key = sync_helpers._get_metadata_key(self._checksum_type)
metadata[metadata_key] = actual_checksum
content, multipart_boundary = construct_multipart_request(
data, metadata, content_type)
multipart_content_type = _RELATED_HEADER + multipart_boundary + b'"'
self._headers[_CONTENT_TYPE_HEADER] = multipart_content_type
return _POST, self.upload_url, content, self._headers
def test__prepare_request_with_checksum(self, checksum):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=checksum)
upload._prepare_request()
assert upload._checksum_object is not None
checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
checksum_digest = sync_helpers.prepare_checksum_digest(
upload._checksum_object.digest())
assert checksum_digest == checksums[checksum]
assert upload._bytes_checksummed == len(data)
def _write_to_stream(self, response):
"""Write response body to a write-able stream.
.. note:
This method assumes that the ``_stream`` attribute is set on the
current download.
Args:
response (~requests.Response): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
"""
# `_get_expected_checksum()` may return None even if a checksum was
# requested, in which case it will emit an info log _MISSING_CHECKSUM.
# If an invalid checksum type is specified, this will raise ValueError.
expected_checksum, checksum_object = _helpers._get_expected_checksum(
response,
self._get_headers,
self.media_url,
checksum_type=self.checksum)
with response:
# NOTE: In order to handle compressed streams gracefully, we try
# to insert our checksum object into the decompression stream. If
# the stream is indeed compressed, this will delegate the checksum
# object to the decoder and return a _DoNothingHash here.
local_checksum_object = _add_decoder(response.raw, checksum_object)
body_iter = response.iter_content(
chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE,
decode_unicode=False)
for chunk in body_iter:
self._stream.write(chunk)
local_checksum_object.update(chunk)
if expected_checksum is None:
return
else:
actual_checksum = _helpers.prepare_checksum_digest(
checksum_object.digest())
if actual_checksum != expected_checksum:
msg = _CHECKSUM_MISMATCH.format(
self.media_url,
expected_checksum,
actual_checksum,
checksum_type=self.checksum.upper(),
)
raise common.DataCorruption(response, msg)
def _write_to_stream(self, response):
"""Write response body to a write-able stream.
.. note:
This method assumes that the ``_stream`` attribute is set on the
current download.
Args:
response (~requests.Response): The HTTP response object.
Raises:
~google.resumable_media.common.DataCorruption: If the download's
checksum doesn't agree with server-computed checksum.
"""
# `_get_expected_checksum()` may return None even if a checksum was
# requested, in which case it will emit an info log _MISSING_CHECKSUM.
# If an invalid checksum type is specified, this will raise ValueError.
expected_checksum, checksum_object = _helpers._get_expected_checksum(
response,
self._get_headers,
self.media_url,
checksum_type=self.checksum)
with response:
body_iter = response.raw.stream(
_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False)
for chunk in body_iter:
self._stream.write(chunk)
checksum_object.update(chunk)
response._content_consumed = True
if expected_checksum is None:
return
else:
actual_checksum = _helpers.prepare_checksum_digest(
checksum_object.digest())
if actual_checksum != expected_checksum:
msg = _CHECKSUM_MISMATCH.format(
self.media_url,
expected_checksum,
actual_checksum,
checksum_type=self.checksum.upper(),
)
raise common.DataCorruption(response, msg)
def test__update_checksum(self, checksum):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=checksum)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 8
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 16
# Continue to the end.
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, len(data), len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == len(data)
checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
checksum_digest = _helpers.prepare_checksum_digest(
upload._checksum_object.digest())
assert checksum_digest == checksums[checksum]
