def test__update_checksum_rewind(self, checksum):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=checksum)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 8
checksum_checkpoint = upload._checksum_object.digest()
# Rewind to the beginning.
upload._stream.seek(0)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 8
assert upload._checksum_object.digest() == checksum_checkpoint
# Rewind but not to the beginning.
upload._stream.seek(4)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == 12
# Continue to the end.
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, len(data), len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == len(data)
checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
checksum_digest = sync_helpers.prepare_checksum_digest(
upload._checksum_object.digest())
assert checksum_digest == checksums[checksum]
def test_exhausted_known_size(self):
data = b"the end"
stream = io.BytesIO(data)
stream.seek(len(data))
with pytest.raises(ValueError) as exc_info:
_upload.get_next_chunk(stream, 1, len(data))
exc_info.match(
u"Stream is already exhausted. There is no content remaining.")
def test_success_known_size_lt_stream_size(self):
data = b"0123456789"
stream = io.BytesIO(data)
chunk_size = 3
total_bytes = len(data) - 2
# Splits into 3 chunks: 012, 345, 67
result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
assert result0 == (0, b"012", u"bytes 0-2/8")
assert result1 == (3, b"345", u"bytes 3-5/8")
assert result2 == (6, b"67", u"bytes 6-7/8")
def test_success_known_size(self):
data = b"0123456789"
stream = io.BytesIO(data)
total_bytes = len(data)
chunk_size = 3
# Splits into 4 chunks: 012, 345, 678, 9
result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
result3 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
assert result0 == (0, b"012", u"bytes 0-2/10")
assert result1 == (3, b"345", u"bytes 3-5/10")
assert result2 == (6, b"678", u"bytes 6-8/10")
assert result3 == (9, b"9", u"bytes 9-9/10")
assert stream.tell() == total_bytes
async def test__validate_checksum_mismatch(self, checksum):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=checksum)
_fix_up_virtual(upload)
# Go ahead and process the entire data in one go for this test.
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, len(data), len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == len(data)
metadata = {
"md5Hash": "ZZZZZZZZZZZZZZZZZZZZZZ==",
"crc32c": "ZZZZZZ==",
}
# This is only used by _validate_checksum for fetching headers and
# logging, so it doesn't need to be fleshed out with a response body.
response = _make_response(headers=metadata)
upload._finished = True
assert upload._checksum_object is not None
# Test passes if it does not raise an error (no assert needed)
with pytest.raises(common.DataCorruption) as exc_info:
await upload._validate_checksum(response)
error = exc_info.value
assert error.response is response
message = error.args[0]
correct_checksums = {
"crc32c": u"Qg8thA==",
"md5": u"GRvfKbqr5klAOwLkxgIf8w=="
}
metadata_key = sync_helpers._get_metadata_key(checksum)
assert message == _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
checksum.upper(), correct_checksums[checksum],
metadata[metadata_key])
async def test__validate_checksum_header_no_match(self, checksum):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=checksum)
_fix_up_virtual(upload)
# Go ahead and process the entire data in one go for this test.
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, len(data), len(data))
upload._update_checksum(start_byte, payload)
assert upload._bytes_checksummed == len(data)
# For this test, each checksum option will be provided with a valid but
# mismatching remote checksum type.
if checksum == "crc32c":
metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w=="}
else:
metadata = {"crc32c": "Qg8thA=="}
# This is only used by _validate_checksum for fetching headers and
# logging, so it doesn't need to be fleshed out with a response body.
response = _make_response(headers=metadata)
upload._finished = True
assert upload._checksum_object is not None
with pytest.raises(common.InvalidResponse) as exc_info:
await upload._validate_checksum(response)
error = exc_info.value
assert error.response is response
message = error.args[0]
metadata_key = sync_helpers._get_metadata_key(checksum)
assert (
message == _upload._UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE
.format(metadata_key))
def test__update_checksum_invalid(self):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum="invalid")
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
with pytest.raises(ValueError):
upload._update_checksum(start_byte, payload)
def test__update_checksum_none(self):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(data, checksum=None)
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, 8, len(data))
upload._update_checksum(start_byte, payload)
assert upload._checksum_object is None
def test_success_unknown_size(self):
data = b"abcdefghij"
stream = io.BytesIO(data)
chunk_size = 6
# Splits into 4 chunks: abcdef, ghij
result0 = _upload.get_next_chunk(stream, chunk_size, None)
result1 = _upload.get_next_chunk(stream, chunk_size, None)
assert result0 == (0, b"abcdef", u"bytes 0-5/*")
assert result1 == (chunk_size, b"ghij", u"bytes 6-9/10")
assert stream.tell() == len(data)
# Do the same when the chunk size evenly divides len(data)
stream.seek(0)
chunk_size = len(data)
# Splits into 2 chunks: `data` and empty string
result0 = _upload.get_next_chunk(stream, chunk_size, None)
result1 = _upload.get_next_chunk(stream, chunk_size, None)
assert result0 == (0, data, u"bytes 0-9/*")
assert result1 == (len(data), b"", u"bytes */10")
assert stream.tell() == len(data)
async def test__validate_checksum_none(self):
data = b"All of the data goes in a stream."
upload = self._upload_in_flight(b"test", checksum=None)
_fix_up_virtual(upload)
# Go ahead and process the entire data in one go for this test.
start_byte, payload, _ = _upload.get_next_chunk(
upload._stream, len(data), len(data))
upload._update_checksum(start_byte, payload)
# This is only used by _validate_checksum for fetching metadata and
# logging.
metadata = {
"md5Hash": "GRvfKbqr5klAOwLkxgIf8w==",
"crc32c": "Qg8thA=="
}
response = _make_response(headers=metadata)
upload._finished = True
assert upload._checksum_object is None
assert upload._bytes_checksummed == 0
# Test passes if it does not raise an error (no assert needed)
await upload._validate_checksum(response)
def test_exhausted_known_size_zero_nonempty(self):
stream = io.BytesIO(b"not empty WAT!")
with pytest.raises(ValueError) as exc_info:
_upload.get_next_chunk(stream, 1, 0)
exc_info.match(
u"Stream specified as empty, but produced non-empty content.")
def test_exhausted_known_size_zero(self):
stream = io.BytesIO(b"")
answer = _upload.get_next_chunk(stream, 1, 0)
assert answer == (0, b"", "bytes */0")