def _consume_helper(
self, stream=None, end=65536, headers=None, chunks=(), response_headers=None
):
download = download_mod.RawDownload(
EXAMPLE_URL, stream=stream, end=end, headers=headers
)
transport = mock.Mock(spec=["request"])
transport.request.return_value = _mock_raw_response(
chunks=chunks, headers=response_headers
)
assert not download.finished
ret_val = download.consume(transport)
assert ret_val is transport.request.return_value
if chunks:
assert stream is not None
transport.request.assert_called_once_with(
u"GET",
EXAMPLE_URL,
data=None,
headers=download._headers,
stream=True,
timeout=EXPECTED_TIMEOUT,
)
range_bytes = u"bytes={:d}-{:d}".format(0, end)
assert download._headers[u"range"] == range_bytes
assert download.finished
return transport
def test__write_to_stream_with_hash_check_fail(self):
stream = io.BytesIO()
download = download_mod.RawDownload(EXAMPLE_URL, stream=stream)
chunk1 = b"first chunk, count starting at 0. "
chunk2 = b"second chunk, or chunk 1, which is better? "
chunk3 = b"ordinals and numerals and stuff."
bad_checksum = u"d3JvbmcgbiBtYWRlIHVwIQ=="
header_value = u"crc32c=V0FUPw==,md5={}".format(bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
with pytest.raises(common.DataCorruption) as exc_info:
download._write_to_stream(response)
assert not download.finished
error = exc_info.value
assert error.response is response
assert len(error.args) == 1
good_checksum = u"fPAJHnnoi/+NadyNxT2c2w=="
msg = download_mod._CHECKSUM_MISMATCH.format(
EXAMPLE_URL, bad_checksum, good_checksum
)
assert error.args[0] == msg
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.raw.stream.assert_called_once_with(
_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
)
def test__write_to_stream_with_invalid_checksum_type(self):
BAD_CHECKSUM_TYPE = "badsum"
stream = io.BytesIO()
download = download_mod.RawDownload(EXAMPLE_URL,
stream=stream,
checksum=BAD_CHECKSUM_TYPE)
chunk1 = b"first chunk, count starting at 0. "
chunk2 = b"second chunk, or chunk 1, which is better? "
chunk3 = b"ordinals and numerals and stuff."
bad_checksum = u"d3JvbmcgbiBtYWRlIHVwIQ=="
header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_response(chunks=[chunk1, chunk2, chunk3],
headers=headers)
with pytest.raises(ValueError) as exc_info:
download._write_to_stream(response)
assert not download.finished
error = exc_info.value
assert error.args[
0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
def test_consume_with_stream_hash_check_fail(self, checksum):
stream = io.BytesIO()
download = download_mod.RawDownload(EXAMPLE_URL,
stream=stream,
checksum=checksum)
chunks = (b"zero zero", b"niner tango")
bad_checksum = u"anVzdCBub3QgdGhpcyAxLA=="
header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
transport = mock.Mock(spec=["request"])
transport.request.return_value = _mock_raw_response(chunks=chunks,
headers=headers)
assert not download.finished
with pytest.raises(common.DataCorruption) as exc_info:
download.consume(transport)
assert stream.getvalue() == b"".join(chunks)
assert download.finished
assert download._headers == {}
error = exc_info.value
assert error.response is transport.request.return_value
assert len(error.args) == 1
if checksum == u"md5":
good_checksum = u"1A/dxEpys717C6FH7FIWDw=="
else:
good_checksum = u"GvNZlg=="
msg = download_mod._CHECKSUM_MISMATCH.format(
EXAMPLE_URL,
bad_checksum,
good_checksum,
checksum_type=checksum.upper())
assert error.args[0] == msg
# Check mocks.
transport.request.assert_called_once_with(
u"GET",
EXAMPLE_URL,
data=None,
headers={},
stream=True,
timeout=EXPECTED_TIMEOUT,
)
def test__write_to_stream_no_hash_check(self):
stream = io.BytesIO()
download = download_mod.RawDownload(EXAMPLE_URL, stream=stream)
chunk1 = b"right now, "
chunk2 = b"but a little later"
response = _mock_raw_response(chunks=[chunk1, chunk2], headers={})
ret_val = download._write_to_stream(response)
assert ret_val is None
assert stream.getvalue() == chunk1 + chunk2
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.raw.stream.assert_called_once_with(
_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False)
def test__write_to_stream_with_hash_check_success(self):
stream = io.BytesIO()
download = download_mod.RawDownload(EXAMPLE_URL, stream=stream)
chunk1 = b"first chunk, count starting at 0. "
chunk2 = b"second chunk, or chunk 1, which is better? "
chunk3 = b"ordinals and numerals and stuff."
header_value = u"crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w=="
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
ret_val = download._write_to_stream(response)
assert ret_val is None
assert stream.getvalue() == chunk1 + chunk2 + chunk3
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.raw.stream.assert_called_once_with(
_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
)
