def test__update_checksum_rewind(self, checksum): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=checksum) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 8 checksum_checkpoint = upload._checksum_object.digest() # Rewind to the beginning. upload._stream.seek(0) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 8 assert upload._checksum_object.digest() == checksum_checkpoint # Rewind but not to the beginning. upload._stream.seek(4) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 12 # Continue to the end. start_byte, payload, _ = _upload.get_next_chunk( upload._stream, len(data), len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == len(data) checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="} checksum_digest = sync_helpers.prepare_checksum_digest( upload._checksum_object.digest()) assert checksum_digest == checksums[checksum]
async def _validate_checksum(self, response): """Check the computed checksum, if any, against the response headers. Args: response (object): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the checksum computed locally and the checksum reported by the remote host do not match. """ if self._checksum_type is None: return metadata_key = sync_helpers._get_metadata_key(self._checksum_type) metadata = await response.json() remote_checksum = metadata.get(metadata_key) if remote_checksum is None: raise common.InvalidResponse( response, _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format( metadata_key), self._get_headers(response), ) local_checksum = sync_helpers.prepare_checksum_digest( self._checksum_object.digest()) if local_checksum != remote_checksum: raise common.DataCorruption( response, _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( self._checksum_type.upper(), local_checksum, remote_checksum), )
def test_multipart_upload_with_bad_checksum(authorized_transport, checksum, bucket): with open(ICO_FILE, u"rb") as file_obj: actual_contents = file_obj.read() blob_name = os.path.basename(ICO_FILE) check_does_not_exist(authorized_transport, blob_name) # Create the actual upload object. upload_url = utils.MULTIPART_UPLOAD upload = resumable_requests.MultipartUpload(upload_url, checksum=checksum) # Transmit the resource. metadata = {u"name": blob_name, u"metadata": {u"color": u"yellow"}} fake_checksum_object = _helpers._get_checksum_object(checksum) fake_checksum_object.update(b"bad data") fake_prepared_checksum_digest = _helpers.prepare_checksum_digest( fake_checksum_object.digest()) with mock.patch.object(_helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest): with pytest.raises(common.InvalidResponse) as exc_info: response = upload.transmit(authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE) response = exc_info.value.response message = response.json()["error"]["message"] # Attempt to verify that this is a checksum mismatch error. assert checksum.upper() in message assert fake_prepared_checksum_digest in message # Make sure the upload is tombstoned. check_tombstoned(upload, authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE)
async def test_resumable_upload_with_bad_checksum(authorized_transport, img_stream, bucket, cleanup, checksum): fake_checksum_object = _helpers._get_checksum_object(checksum) fake_checksum_object.update(b"bad data") fake_prepared_checksum_digest = _helpers.prepare_checksum_digest( fake_checksum_object.digest()) with mock.patch.object(_helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest): with pytest.raises(common.DataCorruption) as exc_info: await _resumable_upload_helper(authorized_transport, img_stream, cleanup, checksum=checksum) expected_checksums = { "md5": "1bsd83IYNug8hd+V1ING3Q==", "crc32c": "YQGPxA==" } expected_message = (_async_resumable_media._upload. _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( checksum.upper(), fake_prepared_checksum_digest, expected_checksums[checksum], )) assert exc_info.value.args[0] == expected_message
def _prepare_request(self, data, metadata, content_type): """Prepare the contents of an HTTP request. This is everything that must be done before a request that doesn't require network I/O (or other I/O). This is based on the `sans-I/O`_ philosophy. .. note: This method will be used only once, so ``headers`` will be mutated by having a new key added to it. Args: data (bytes): The resource content to be uploaded. metadata (Mapping[str, str]): The resource metadata, such as an ACL list. content_type (str): The content type of the resource, e.g. a JPEG image has content type ``image/jpeg``. Returns: Tuple[str, str, bytes, Mapping[str, str]]: The quadruple * HTTP verb for the request (always POST) * the URL for the request * the body of the request * headers for the request Raises: ValueError: If the current upload has already finished. TypeError: If ``data`` isn't bytes. .. _sans-I/O: https://sans-io.readthedocs.io/ """ if self.finished: raise ValueError("An upload can only be used once.") if not isinstance(data, bytes): raise TypeError("`data` must be bytes, received", type(data)) checksum_object = sync_helpers._get_checksum_object( self._checksum_type) if checksum_object is not None: checksum_object.update(data) actual_checksum = sync_helpers.prepare_checksum_digest( checksum_object.digest()) metadata_key = sync_helpers._get_metadata_key(self._checksum_type) metadata[metadata_key] = actual_checksum content, multipart_boundary = construct_multipart_request( data, metadata, content_type) multipart_content_type = _RELATED_HEADER + multipart_boundary + b'"' self._headers[_CONTENT_TYPE_HEADER] = multipart_content_type return _POST, self.upload_url, content, self._headers
def test__prepare_request_with_checksum(self, checksum): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=checksum) upload._prepare_request() assert upload._checksum_object is not None checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="} checksum_digest = sync_helpers.prepare_checksum_digest( upload._checksum_object.digest()) assert checksum_digest == checksums[checksum] assert upload._bytes_checksummed == len(data)
def _write_to_stream(self, response): """Write response body to a write-able stream. .. note: This method assumes that the ``_stream`` attribute is set on the current download. Args: response (~requests.Response): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ # `_get_expected_checksum()` may return None even if a checksum was # requested, in which case it will emit an info log _MISSING_CHECKSUM. # If an invalid checksum type is specified, this will raise ValueError. expected_checksum, checksum_object = _helpers._get_expected_checksum( response, self._get_headers, self.media_url, checksum_type=self.checksum) with response: # NOTE: In order to handle compressed streams gracefully, we try # to insert our checksum object into the decompression stream. If # the stream is indeed compressed, this will delegate the checksum # object to the decoder and return a _DoNothingHash here. local_checksum_object = _add_decoder(response.raw, checksum_object) body_iter = response.iter_content( chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False) for chunk in body_iter: self._stream.write(chunk) local_checksum_object.update(chunk) if expected_checksum is None: return else: actual_checksum = _helpers.prepare_checksum_digest( checksum_object.digest()) if actual_checksum != expected_checksum: msg = _CHECKSUM_MISMATCH.format( self.media_url, expected_checksum, actual_checksum, checksum_type=self.checksum.upper(), ) raise common.DataCorruption(response, msg)
def _write_to_stream(self, response): """Write response body to a write-able stream. .. note: This method assumes that the ``_stream`` attribute is set on the current download. Args: response (~requests.Response): The HTTP response object. Raises: ~google.resumable_media.common.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ # `_get_expected_checksum()` may return None even if a checksum was # requested, in which case it will emit an info log _MISSING_CHECKSUM. # If an invalid checksum type is specified, this will raise ValueError. expected_checksum, checksum_object = _helpers._get_expected_checksum( response, self._get_headers, self.media_url, checksum_type=self.checksum) with response: body_iter = response.raw.stream( _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False) for chunk in body_iter: self._stream.write(chunk) checksum_object.update(chunk) response._content_consumed = True if expected_checksum is None: return else: actual_checksum = _helpers.prepare_checksum_digest( checksum_object.digest()) if actual_checksum != expected_checksum: msg = _CHECKSUM_MISMATCH.format( self.media_url, expected_checksum, actual_checksum, checksum_type=self.checksum.upper(), ) raise common.DataCorruption(response, msg)
def test__update_checksum(self, checksum): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=checksum) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 8 start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 16 # Continue to the end. start_byte, payload, _ = _upload.get_next_chunk( upload._stream, len(data), len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == len(data) checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="} checksum_digest = _helpers.prepare_checksum_digest( upload._checksum_object.digest()) assert checksum_digest == checksums[checksum]