def test_exhausted_known_size_zero_nonempty(self): stream = io.BytesIO(b'not empty WAT!') with pytest.raises(ValueError) as exc_info: _upload.get_next_chunk(stream, 1, 0) exc_info.match( u'Stream specified as empty, but produced non-empty content.')
def test__update_checksum_rewind(self, checksum): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=checksum) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 8 checksum_checkpoint = upload._checksum_object.digest() # Rewind to the beginning. upload._stream.seek(0) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 8 assert upload._checksum_object.digest() == checksum_checkpoint # Rewind but not to the beginning. upload._stream.seek(4) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == 12 # Continue to the end. start_byte, payload, _ = _upload.get_next_chunk( upload._stream, len(data), len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == len(data) checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="} checksum_digest = _helpers.prepare_checksum_digest( upload._checksum_object.digest()) assert checksum_digest == checksums[checksum]
def test_exhausted_known_size(self): data = b"the end" stream = io.BytesIO(data) stream.seek(len(data)) with pytest.raises(ValueError) as exc_info: _upload.get_next_chunk(stream, 1, len(data)) exc_info.match(u"Stream is already exhausted. There is no content remaining.")
def test_read_past_known_size(self): data = b"more content than we expected" stream = io.BytesIO(data) chunk_size = len(data) total_bytes = chunk_size - 3 with pytest.raises(ValueError) as exc_info: _upload.get_next_chunk(stream, chunk_size, total_bytes) exc_info.match(u"bytes have been read from the stream") exc_info.match(u"exceeds the expected total")
def test_success_known_size_lt_stream_size(self): data = b"0123456789" stream = io.BytesIO(data) chunk_size = 3 total_bytes = len(data) - 2 # Splits into 3 chunks: 012, 345, 67 result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes) result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes) result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes) assert result0 == (0, b"012", u"bytes 0-2/8") assert result1 == (3, b"345", u"bytes 3-5/8") assert result2 == (6, b"67", u"bytes 6-7/8")
def test_success_known_size(self): data = b"0123456789" stream = io.BytesIO(data) total_bytes = len(data) chunk_size = 3 # Splits into 4 chunks: 012, 345, 678, 9 result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes) result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes) result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes) result3 = _upload.get_next_chunk(stream, chunk_size, total_bytes) assert result0 == (0, b"012", u"bytes 0-2/10") assert result1 == (3, b"345", u"bytes 3-5/10") assert result2 == (6, b"678", u"bytes 6-8/10") assert result3 == (9, b"9", u"bytes 9-9/10") assert stream.tell() == total_bytes
def test__validate_checksum_mismatch(self, checksum): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=checksum) _fix_up_virtual(upload) # Go ahead and process the entire data in one go for this test. start_byte, payload, _ = _upload.get_next_chunk( upload._stream, len(data), len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == len(data) metadata = { "md5Hash": "ZZZZZZZZZZZZZZZZZZZZZZ==", "crc32c": "ZZZZZZ==", } # This is only used by _validate_checksum for fetching headers and # logging, so it doesn't need to be fleshed out with a response body. response = _make_response(metadata=metadata) upload._finished = True assert upload._checksum_object is not None # Test passes if it does not raise an error (no assert needed) with pytest.raises(common.DataCorruption) as exc_info: upload._validate_checksum(response) error = exc_info.value assert error.response is response message = error.args[0] correct_checksums = { "crc32c": u"Qg8thA==", "md5": u"GRvfKbqr5klAOwLkxgIf8w==" } metadata_key = _helpers._get_metadata_key(checksum) assert message == _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( checksum.upper(), correct_checksums[checksum], metadata[metadata_key])
def test__validate_checksum_header_no_match(self, checksum): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=checksum) _fix_up_virtual(upload) # Go ahead and process the entire data in one go for this test. start_byte, payload, _ = _upload.get_next_chunk( upload._stream, len(data), len(data)) upload._update_checksum(start_byte, payload) assert upload._bytes_checksummed == len(data) # For this test, each checksum option will be provided with a valid but # mismatching remote checksum type. if checksum == "crc32c": metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w=="} else: metadata = {"crc32c": "Qg8thA=="} # This is only used by _validate_checksum for fetching headers and # logging, so it doesn't need to be fleshed out with a response body. response = _make_response(metadata=metadata) upload._finished = True assert upload._checksum_object is not None with pytest.raises(common.InvalidResponse) as exc_info: upload._validate_checksum(response) error = exc_info.value assert error.response is response message = error.args[0] metadata_key = _helpers._get_metadata_key(checksum) assert ( message == _upload._UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE .format(metadata_key))
def test__update_checksum_invalid(self): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum="invalid") start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) with pytest.raises(ValueError): upload._update_checksum(start_byte, payload)
def test__update_checksum_none(self): data = b"All of the data goes in a stream." upload = self._upload_in_flight(data, checksum=None) start_byte, payload, _ = _upload.get_next_chunk( upload._stream, 8, len(data)) upload._update_checksum(start_byte, payload) assert upload._checksum_object is None
def test_success_unknown_size(self): data = b"abcdefghij" stream = io.BytesIO(data) chunk_size = 6 # Splits into 4 chunks: abcdef, ghij result0 = _upload.get_next_chunk(stream, chunk_size, None) result1 = _upload.get_next_chunk(stream, chunk_size, None) assert result0 == (0, b"abcdef", u"bytes 0-5/*") assert result1 == (chunk_size, b"ghij", u"bytes 6-9/10") assert stream.tell() == len(data) # Do the same when the chunk size evenly divides len(data) stream.seek(0) chunk_size = len(data) # Splits into 2 chunks: `data` and empty string result0 = _upload.get_next_chunk(stream, chunk_size, None) result1 = _upload.get_next_chunk(stream, chunk_size, None) assert result0 == (0, data, u"bytes 0-9/*") assert result1 == (len(data), b"", u"bytes */10") assert stream.tell() == len(data)
def test__validate_checksum_none(self): data = b"All of the data goes in a stream." upload = self._upload_in_flight(b"test", checksum=None) _fix_up_virtual(upload) # Go ahead and process the entire data in one go for this test. start_byte, payload, _ = _upload.get_next_chunk( upload._stream, len(data), len(data)) upload._update_checksum(start_byte, payload) # This is only used by _validate_checksum for fetching metadata and # logging. metadata = { "md5Hash": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA==" } response = _make_response(metadata=metadata) upload._finished = True assert upload._checksum_object is None assert upload._bytes_checksummed == 0 # Test passes if it does not raise an error (no assert needed) upload._validate_checksum(response)
def test_exhausted_known_size_zero(self): stream = io.BytesIO(b"") answer = _upload.get_next_chunk(stream, 1, 0) assert answer == (0, b"", "bytes */0")