def test__process_response_bad_status(self):
chunk_size = 384
stream = mock.Mock(spec=[u"write"])
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
_fix_up_virtual(download)
total_bytes = 300
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
# Actually call the method to update.
response = self._mock_response(
0, total_bytes - 1, total_bytes, status_code=int(http_client.NOT_FOUND)
)
with pytest.raises(common.InvalidResponse) as exc_info:
download._process_response(response)
error = exc_info.value
assert error.response is response
assert len(error.args) == 5
assert error.args[1] == response.status_code
assert error.args[3] == http_client.OK
assert error.args[4] == http_client.PARTIAL_CONTENT
# Check internal state after.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
assert download.invalid
stream.write.assert_not_called()
def test__process_response_bad_content_range(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
_fix_up_virtual(download)
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
assert not download.invalid
# Actually call the method to update.
data = b"stuff"
headers = {
u"content-length": u"{:d}".format(len(data)),
u"content-range": u"kites x-y/58",
}
response = mock.Mock(
content=data,
headers=headers,
status_code=int(http_client.PARTIAL_CONTENT),
spec=[u"content", u"headers", u"status_code"],
)
with pytest.raises(common.InvalidResponse) as exc_info:
download._process_response(response)
error = exc_info.value
assert error.response is response
assert len(error.args) == 3
assert error.args[1] == headers[u"content-range"]
# Check internal state after.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
assert download.invalid
def test__process_response(self):
chunk_size = 333
stream = io.BytesIO()
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
_fix_up_virtual(download)
already = 22
download._bytes_downloaded = already
total_bytes = 4444
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == already
assert download.total_bytes is None
# Actually call the method to update.
data = b"1234xyztL" * 37 # 9 * 37 == 33
response = self._mock_response(
already,
already + chunk_size - 1,
total_bytes,
content=data,
status_code=int(http_client.PARTIAL_CONTENT),
)
download._process_response(response)
# Check internal state after.
assert not download.finished
assert download.bytes_downloaded == already + chunk_size
assert download.total_bytes == total_bytes
assert stream.getvalue() == data
def test__process_response_transfer_encoding(self):
data = b'1234xyztL' * 37
chunk_size = len(data)
stream = io.BytesIO()
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
_fix_up_virtual(download)
already = 22
download._bytes_downloaded = already
total_bytes = 4444
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == already
assert download.total_bytes is None
assert not download.invalid
# Actually call the method to update.
response = self._mock_response(already,
already + chunk_size - 1,
total_bytes,
content=data,
status_code=int(
http_client.PARTIAL_CONTENT))
response.headers[u'transfer-encoding'] = 'chunked'
del response.headers[u'content-length']
download._process_response(response)
# Check internal state after.
assert not download.finished
assert download.bytes_downloaded == already + chunk_size
assert download.total_bytes == total_bytes
assert stream.getvalue() == data
def test__prepare_request_already_finished(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 64, None)
download._finished = True
with pytest.raises(ValueError) as exc_info:
download._prepare_request()
assert exc_info.match(u"Download has finished.")
def test__prepare_request_invalid(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 64, None)
download._invalid = True
with pytest.raises(ValueError) as exc_info:
download._prepare_request()
assert exc_info.match(u"Download is invalid and cannot be re-used.")
def test__process_response_when_finished(self):
chunk_size = 256
stream = io.BytesIO()
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
_fix_up_virtual(download)
total_bytes = 200
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
# Actually call the method to update.
data = b"abcd" * 50 # 4 * 50 == 200
response = self._mock_response(
0,
total_bytes - 1,
total_bytes,
content=data,
status_code=int(http_client.OK),
)
download._process_response(response)
# Check internal state after.
assert download.finished
assert download.bytes_downloaded == total_bytes
assert total_bytes < chunk_size
assert download.total_bytes == total_bytes
assert stream.getvalue() == data
def test__process_response_body_wrong_length(self):
chunk_size = 10
stream = mock.Mock(spec=[u"write"])
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
_fix_up_virtual(download)
total_bytes = 100
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
# Actually call the method to update.
data = b"not 10"
response = self._mock_response(
0,
chunk_size - 1,
total_bytes,
content=data,
status_code=int(http_client.PARTIAL_CONTENT),
)
with pytest.raises(common.InvalidResponse) as exc_info:
download._process_response(response)
error = exc_info.value
assert error.response is response
assert len(error.args) == 5
assert error.args[2] == chunk_size
assert error.args[4] == len(data)
# Check internal state after.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
assert download.invalid
stream.write.assert_not_called()
def test__process_response_when_reaching_end(self):
chunk_size = 8192
end = 65000
stream = io.BytesIO()
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream, end=end)
_fix_up_virtual(download)
download._bytes_downloaded = 7 * chunk_size
download._total_bytes = 8 * chunk_size
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == 7 * chunk_size
assert download.total_bytes == 8 * chunk_size
# Actually call the method to update.
expected_size = end - 7 * chunk_size + 1
data = b"B" * expected_size
response = self._mock_response(
7 * chunk_size,
end,
8 * chunk_size,
content=data,
status_code=int(http_client.PARTIAL_CONTENT),
)
download._process_response(response)
# Check internal state after.
assert download.finished
assert download.bytes_downloaded == end + 1
assert download.bytes_downloaded < download.total_bytes
assert download.total_bytes == 8 * chunk_size
assert stream.getvalue() == data
def test__process_response_missing_content_length(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
_fix_up_virtual(download)
# Check internal state before.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
assert not download.invalid
# Actually call the method to update.
response = mock.Mock(
headers={},
status_code=int(http_client.PARTIAL_CONTENT),
spec=[u"headers", u"status_code"],
)
with pytest.raises(common.InvalidResponse) as exc_info:
download._process_response(response)
error = exc_info.value
assert error.response is response
assert len(error.args) == 2
assert error.args[1] == u"content-length"
# Check internal state after.
assert not download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
assert download.invalid
def test__prepare_request(self):
chunk_size = 2048
download1 = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
method1, url1, payload1, headers1 = download1._prepare_request()
assert method1 == u"GET"
assert url1 == EXAMPLE_URL
assert payload1 is None
assert headers1 == {u"range": u"bytes=0-2047"}
download2 = _download.ChunkedDownload(
EXAMPLE_URL, chunk_size, None, start=19991
)
download2._total_bytes = 20101
method2, url2, payload2, headers2 = download2._prepare_request()
assert method2 == u"GET"
assert url2 == EXAMPLE_URL
assert payload2 is None
assert headers2 == {u"range": u"bytes=19991-20100"}
def test__get_byte_range_with_total_bytes(self):
chunk_size = 512
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
total_bytes = 207
download._total_bytes = total_bytes
curr_start, curr_end = download._get_byte_range()
assert curr_start == 0
assert curr_end == total_bytes - 1
# Make sure this is less than the chunk size.
actual_size = curr_end - curr_start + 1
assert actual_size < chunk_size
def test_total_bytes_property(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
# Default value of @property.
assert download.total_bytes is None
# Make sure we cannot set it on public @property.
with pytest.raises(AttributeError):
download.total_bytes = 65536
# Set it privately and then check the @property.
download._total_bytes = 8192
assert download.total_bytes == 8192
def test_bytes_downloaded_property(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
# Default value of @property.
assert download.bytes_downloaded == 0
# Make sure we cannot set it on public @property.
with pytest.raises(AttributeError):
download.bytes_downloaded = 1024
# Set it privately and then check the @property.
download._bytes_downloaded = 128
assert download.bytes_downloaded == 128
def test__prepare_request_with_headers(self):
chunk_size = 2048
headers = {u"patrizio": u"Starf-ish"}
download = _download.ChunkedDownload(
EXAMPLE_URL, chunk_size, None, headers=headers
)
method, url, payload, new_headers = download._prepare_request()
assert method == u"GET"
assert url == EXAMPLE_URL
assert payload is None
assert new_headers is headers
expected = {u"patrizio": u"Starf-ish", u"range": u"bytes=0-2047"}
assert headers == expected
def test__get_byte_range_with_end(self):
chunk_size = 512
start = 1024
end = 1151
download = _download.ChunkedDownload(
EXAMPLE_URL, chunk_size, None, start=start, end=end
)
curr_start, curr_end = download._get_byte_range()
assert curr_start == start
assert curr_end == end
# Make sure this is less than the chunk size.
actual_size = curr_end - curr_start + 1
assert actual_size < chunk_size
def test__prepare_request_with_headers(self):
chunk_size = 2048
headers = {u'patrizio': u'Starf-ish'}
download = _download.ChunkedDownload(EXAMPLE_URL,
chunk_size,
None,
headers=headers)
method, url, payload, new_headers = download._prepare_request()
assert method == u'GET'
assert url == EXAMPLE_URL
assert payload is None
assert new_headers is headers
expected = {u'patrizio': u'Starf-ish', u'range': u'bytes=0-2047'}
assert headers == expected
def test_constructor_defaults(self):
chunk_size = 256
stream = mock.sentinel.stream
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
assert download.media_url == EXAMPLE_URL
assert download.chunk_size == chunk_size
assert download.start == 0
assert download.end is None
assert download._headers == {}
assert not download._finished
_check_retry_strategy(download)
assert download._stream is stream
assert download._bytes_downloaded == 0
assert download._total_bytes is None
assert not download._invalid
def test__process_response_when_content_range_is_zero(self):
chunk_size = 10
stream = mock.Mock(spec=[u"write"])
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
_fix_up_virtual(download)
content_range = _download._ZERO_CONTENT_RANGE_HEADER
headers = {u"content-range": content_range}
status_code = http_client.REQUESTED_RANGE_NOT_SATISFIABLE
response = mock.Mock(
headers=headers, status_code=status_code, spec=[u"headers", "status_code"]
)
download._process_response(response)
stream.write.assert_not_called()
assert download.finished
assert download.bytes_downloaded == 0
assert download.total_bytes is None
def test_consume_next_chunk(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
with pytest.raises(NotImplementedError) as exc_info:
download.consume_next_chunk(None)
exc_info.match(u"virtual")
def test_constructor_bad_start(self):
with pytest.raises(ValueError):
_download.ChunkedDownload(EXAMPLE_URL, 256, None, start=-11)
def test__get_byte_range(self):
chunk_size = 512
download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
curr_start, curr_end = download._get_byte_range()
assert curr_start == 0
assert curr_end == chunk_size - 1
def test__make_invalid(self):
download = _download.ChunkedDownload(EXAMPLE_URL, 512, None)
assert not download.invalid
download._make_invalid()
assert download.invalid
