def test_consume_with_stream_hash_check_fail(self):
stream = io.BytesIO()
download = download_mod.Download(
EXAMPLE_URL, stream=stream)
chunks = (b'zero zero', b'niner tango')
bad_checksum = u'anVzdCBub3QgdGhpcyAxLA=='
header_value = u'crc32c=V0FUPw==,md5={}'.format(bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
transport = mock.Mock(spec=[u'request'])
transport.request.return_value = _mock_response(
chunks=chunks, headers=headers)
assert not download.finished
with pytest.raises(common.DataCorruption) as exc_info:
download.consume(transport)
assert stream.getvalue() == b''.join(chunks)
assert download.finished
assert download._headers == {}
error = exc_info.value
assert error.response is transport.request.return_value
assert len(error.args) == 1
good_checksum = u'1A/dxEpys717C6FH7FIWDw=='
msg = download_mod._CHECKSUM_MISMATCH.format(
EXAMPLE_URL, bad_checksum, good_checksum)
assert error.args[0] == msg
# Check mocks.
transport.request.assert_called_once_with(
u'GET', EXAMPLE_URL, data=None, headers={}, stream=True)
def test__write_to_stream_with_hash_check_fail(self):
stream = io.BytesIO()
download = download_mod.Download(EXAMPLE_URL, stream=stream)
chunk1 = b'first chunk, count starting at 0. '
chunk2 = b'second chunk, or chunk 1, which is better? '
chunk3 = b'ordinals and numerals and stuff.'
bad_checksum = u'd3JvbmcgbiBtYWRlIHVwIQ=='
header_value = u'crc32c=V0FUPw==,md5={}'.format(bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_response(
chunks=[chunk1, chunk2, chunk3], headers=headers)
with pytest.raises(common.DataCorruption) as exc_info:
download._write_to_stream(response)
assert not download.finished
error = exc_info.value
assert error.response is response
assert len(error.args) == 1
good_checksum = u'fPAJHnnoi/+NadyNxT2c2w=='
msg = download_mod._CHECKSUM_MISMATCH.format(
EXAMPLE_URL, bad_checksum, good_checksum)
assert error.args[0] == msg
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.iter_content.assert_called_once_with(
chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE,
decode_unicode=False)
def _consume_helper(
self, stream=None, end=65536, headers=None, chunks=(),
response_headers=None):
download = download_mod.Download(
EXAMPLE_URL, stream=stream, end=end, headers=headers)
transport = mock.Mock(spec=[u'request'])
transport.request.return_value = _mock_response(
chunks=chunks, headers=response_headers)
assert not download.finished
ret_val = download.consume(transport)
assert ret_val is transport.request.return_value
called_kwargs = {u'data': None, u'headers': download._headers}
if chunks:
assert stream is not None
called_kwargs[u'stream'] = True
transport.request.assert_called_once_with(
u'GET', EXAMPLE_URL, **called_kwargs)
range_bytes = u'bytes={:d}-{:d}'.format(0, end)
assert download._headers[u'range'] == range_bytes
assert download.finished
return transport
def _consume_helper(
self, stream=None, end=65536, headers=None, chunks=(),
response_headers=None):
download = download_mod.Download(
EXAMPLE_URL, stream=stream, end=end, headers=headers)
transport = mock.Mock(spec=['request'])
transport.request.return_value = _mock_response(
chunks=chunks, headers=response_headers)
assert not download.finished
ret_val = download.consume(transport)
assert ret_val is transport.request.return_value
if chunks:
assert stream is not None
transport.request.assert_called_once_with(
u'GET',
EXAMPLE_URL,
data=None,
headers=download._headers,
stream=True,
timeout=EXPECTED_TIMEOUT,
)
range_bytes = u'bytes={:d}-{:d}'.format(0, end)
assert download._headers[u'range'] == range_bytes
assert download.finished
return transport
def _consume_helper(
self, stream=None, end=65536, headers=None, chunks=(), response_headers=None
):
download = download_mod.Download(
EXAMPLE_URL, stream=stream, end=end, headers=headers
)
transport = mock.Mock(spec=["request"])
transport.request.return_value = _mock_response(
chunks=chunks, headers=response_headers
)
assert not download.finished
ret_val = download.consume(transport)
assert ret_val is transport.request.return_value
called_kwargs = {u"data": None, u"headers": download._headers}
if chunks:
assert stream is not None
called_kwargs[u"stream"] = True
transport.request.assert_called_once_with(
u"GET", EXAMPLE_URL, timeout=EXPECTED_TIMEOUT, **called_kwargs
)
range_bytes = u"bytes={:d}-{:d}".format(0, end)
assert download._headers[u"range"] == range_bytes
assert download.finished
return transport
def test__write_to_stream_with_hash_check_fail(self, checksum):
stream = io.BytesIO()
download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum)
chunk1 = b"first chunk, count starting at 0. "
chunk2 = b"second chunk, or chunk 1, which is better? "
chunk3 = b"ordinals and numerals and stuff."
bad_checksum = u"d3JvbmcgbiBtYWRlIHVwIQ=="
header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum)
headers = {_helpers._HASH_HEADER: header_value}
response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
with pytest.raises(common.DataCorruption) as exc_info:
download._write_to_stream(response)
assert not download.finished
error = exc_info.value
assert error.response is response
assert len(error.args) == 1
if checksum == u"md5":
good_checksum = u"fPAJHnnoi/+NadyNxT2c2w=="
else:
good_checksum = u"qmNCyg=="
msg = download_mod._CHECKSUM_MISMATCH.format(
EXAMPLE_URL, bad_checksum, good_checksum, checksum_type=checksum.upper()
)
assert error.args[0] == msg
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.iter_content.assert_called_once_with(
chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
)
def test__write_to_stream_with_invalid_checksum_type(self):
BAD_CHECKSUM_TYPE = "badsum"
stream = io.BytesIO()
download = download_mod.Download(EXAMPLE_URL,
stream=stream,
checksum=BAD_CHECKSUM_TYPE)
chunk1 = b"first chunk, count starting at 0. "
chunk2 = b"second chunk, or chunk 1, which is better? "
chunk3 = b"ordinals and numerals and stuff."
bad_checksum = u"d3JvbmcgbiBtYWRlIHVwIQ=="
header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_response(chunks=[chunk1, chunk2, chunk3],
headers=headers)
with pytest.raises(ValueError) as exc_info:
download._write_to_stream(response)
assert not download.finished
error = exc_info.value
assert error.args[
0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
def test__get_expected_md5_missing(self, _LOGGER):
download = download_mod.Download(EXAMPLE_URL)
headers = {}
response = _mock_response(headers=headers)
expected_md5_hash = download._get_expected_md5(response)
assert expected_md5_hash is None
expected_msg = download_mod._MISSING_MD5.format(EXAMPLE_URL)
_LOGGER.info.assert_called_once_with(expected_msg)
def test__get_expected_md5_present(self, _LOGGER):
download = download_mod.Download(EXAMPLE_URL)
checksum = u'b2twdXNodGhpc2J1dHRvbg=='
header_value = u'crc32c=3q2+7w==,md5={}'.format(checksum)
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_response(headers=headers)
expected_md5_hash = download._get_expected_md5(response)
assert expected_md5_hash == checksum
_LOGGER.info.assert_not_called()
def test_consume_with_stream_hash_check_fail(self, checksum):
stream = io.BytesIO()
download = download_mod.Download(EXAMPLE_URL,
stream=stream,
checksum=checksum)
chunks = (b"zero zero", b"niner tango")
bad_checksum = u"anVzdCBub3QgdGhpcyAxLA=="
header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum)
headers = {download_mod._HASH_HEADER: header_value}
transport = mock.Mock(spec=["request"])
transport.request.return_value = _mock_response(chunks=chunks,
headers=headers)
assert not download.finished
with pytest.raises(common.DataCorruption) as exc_info:
download.consume(transport)
assert stream.getvalue() == b"".join(chunks)
assert download.finished
assert download._headers == {}
error = exc_info.value
assert error.response is transport.request.return_value
assert len(error.args) == 1
if checksum == u"md5":
good_checksum = u"1A/dxEpys717C6FH7FIWDw=="
else:
good_checksum = u"GvNZlg=="
msg = download_mod._CHECKSUM_MISMATCH.format(
EXAMPLE_URL,
bad_checksum,
good_checksum,
checksum_type=checksum.upper())
assert error.args[0] == msg
# Check mocks.
transport.request.assert_called_once_with(
u"GET",
EXAMPLE_URL,
data=None,
headers={},
stream=True,
timeout=EXPECTED_TIMEOUT,
)
def test__write_to_stream_no_hash_check(self):
stream = io.BytesIO()
download = download_mod.Download(EXAMPLE_URL, stream=stream)
chunk1 = b'right now, '
chunk2 = b'but a little later'
response = _mock_response(chunks=[chunk1, chunk2], headers={})
ret_val = download._write_to_stream(response)
assert ret_val is None
assert stream.getvalue() == chunk1 + chunk2
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.raw.stream.assert_called_once_with(
_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False)
def test__write_to_stream_with_hash_check_success(self):
stream = io.BytesIO()
download = download_mod.Download(EXAMPLE_URL, stream=stream)
chunk1 = b'first chunk, count starting at 0. '
chunk2 = b'second chunk, or chunk 1, which is better? '
chunk3 = b'ordinals and numerals and stuff.'
header_value = u'crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w=='
headers = {download_mod._HASH_HEADER: header_value}
response = _mock_response(
chunks=[chunk1, chunk2, chunk3], headers=headers)
ret_val = download._write_to_stream(response)
assert ret_val is None
assert stream.getvalue() == chunk1 + chunk2 + chunk3
# Check mocks.
response.__enter__.assert_called_once_with()
response.__exit__.assert_called_once_with(None, None, None)
response.raw.stream.assert_called_once_with(
_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False)
