def _consume_helper( self, stream=None, end=65536, headers=None, chunks=(), response_headers=None ): download = download_mod.RawDownload( EXAMPLE_URL, stream=stream, end=end, headers=headers ) transport = mock.Mock(spec=["request"]) transport.request.return_value = _mock_raw_response( chunks=chunks, headers=response_headers ) assert not download.finished ret_val = download.consume(transport) assert ret_val is transport.request.return_value if chunks: assert stream is not None transport.request.assert_called_once_with( u"GET", EXAMPLE_URL, data=None, headers=download._headers, stream=True, timeout=EXPECTED_TIMEOUT, ) range_bytes = u"bytes={:d}-{:d}".format(0, end) assert download._headers[u"range"] == range_bytes assert download.finished return transport
def test__write_to_stream_with_hash_check_fail(self): stream = io.BytesIO() download = download_mod.RawDownload(EXAMPLE_URL, stream=stream) chunk1 = b"first chunk, count starting at 0. " chunk2 = b"second chunk, or chunk 1, which is better? " chunk3 = b"ordinals and numerals and stuff." bad_checksum = u"d3JvbmcgbiBtYWRlIHVwIQ==" header_value = u"crc32c=V0FUPw==,md5={}".format(bad_checksum) headers = {download_mod._HASH_HEADER: header_value} response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers) with pytest.raises(common.DataCorruption) as exc_info: download._write_to_stream(response) assert not download.finished error = exc_info.value assert error.response is response assert len(error.args) == 1 good_checksum = u"fPAJHnnoi/+NadyNxT2c2w==" msg = download_mod._CHECKSUM_MISMATCH.format( EXAMPLE_URL, bad_checksum, good_checksum ) assert error.args[0] == msg # Check mocks. response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.raw.stream.assert_called_once_with( _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False )
def test__write_to_stream_with_invalid_checksum_type(self): BAD_CHECKSUM_TYPE = "badsum" stream = io.BytesIO() download = download_mod.RawDownload(EXAMPLE_URL, stream=stream, checksum=BAD_CHECKSUM_TYPE) chunk1 = b"first chunk, count starting at 0. " chunk2 = b"second chunk, or chunk 1, which is better? " chunk3 = b"ordinals and numerals and stuff." bad_checksum = u"d3JvbmcgbiBtYWRlIHVwIQ==" header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum) headers = {download_mod._HASH_HEADER: header_value} response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers) with pytest.raises(ValueError) as exc_info: download._write_to_stream(response) assert not download.finished error = exc_info.value assert error.args[ 0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
def test_consume_with_stream_hash_check_fail(self, checksum): stream = io.BytesIO() download = download_mod.RawDownload(EXAMPLE_URL, stream=stream, checksum=checksum) chunks = (b"zero zero", b"niner tango") bad_checksum = u"anVzdCBub3QgdGhpcyAxLA==" header_value = u"crc32c={bad},md5={bad}".format(bad=bad_checksum) headers = {download_mod._HASH_HEADER: header_value} transport = mock.Mock(spec=["request"]) transport.request.return_value = _mock_raw_response(chunks=chunks, headers=headers) assert not download.finished with pytest.raises(common.DataCorruption) as exc_info: download.consume(transport) assert stream.getvalue() == b"".join(chunks) assert download.finished assert download._headers == {} error = exc_info.value assert error.response is transport.request.return_value assert len(error.args) == 1 if checksum == u"md5": good_checksum = u"1A/dxEpys717C6FH7FIWDw==" else: good_checksum = u"GvNZlg==" msg = download_mod._CHECKSUM_MISMATCH.format( EXAMPLE_URL, bad_checksum, good_checksum, checksum_type=checksum.upper()) assert error.args[0] == msg # Check mocks. transport.request.assert_called_once_with( u"GET", EXAMPLE_URL, data=None, headers={}, stream=True, timeout=EXPECTED_TIMEOUT, )
def test__write_to_stream_no_hash_check(self): stream = io.BytesIO() download = download_mod.RawDownload(EXAMPLE_URL, stream=stream) chunk1 = b"right now, " chunk2 = b"but a little later" response = _mock_raw_response(chunks=[chunk1, chunk2], headers={}) ret_val = download._write_to_stream(response) assert ret_val is None assert stream.getvalue() == chunk1 + chunk2 # Check mocks. response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.raw.stream.assert_called_once_with( _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False)
def test__write_to_stream_with_hash_check_success(self): stream = io.BytesIO() download = download_mod.RawDownload(EXAMPLE_URL, stream=stream) chunk1 = b"first chunk, count starting at 0. " chunk2 = b"second chunk, or chunk 1, which is better? " chunk3 = b"ordinals and numerals and stuff." header_value = u"crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w==" headers = {download_mod._HASH_HEADER: header_value} response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers) ret_val = download._write_to_stream(response) assert ret_val is None assert stream.getvalue() == chunk1 + chunk2 + chunk3 # Check mocks. response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.raw.stream.assert_called_once_with( _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False )