def test_get_info_expired(self):
utc_now = datetime.datetime.utcnow()
# expires in the past
expired_info = download_threads.PresignedUrlInfo(
"myFile.txt",
"https://synapse.org/somefile.txt",
expiration_utc=utc_now - datetime.timedelta(seconds=5))
unexpired_info = download_threads.PresignedUrlInfo(
"myFile.txt",
"https://synapse.org/somefile.txt",
expiration_utc=utc_now + datetime.timedelta(seconds=6))
with mock.patch.object(download_threads.PresignedUrlProvider, '_get_pre_signed_info',
side_effect=[expired_info, unexpired_info]) as mock_get_presigned_info, \
mock.patch.object(download_threads, "datetime") as mock_datetime:
mock_datetime.datetime.utcnow.return_value = utc_now
presigned_url_provider = download_threads.PresignedUrlProvider(
self.mock_synapse_client, self.download_request)
assert_equals(unexpired_info, presigned_url_provider.get_info())
# only caled once in init and again in get_info
assert_equals(2, mock_get_presigned_info.call_count)
mock_datetime.datetime.utcnow.assert_called_once()
def test_get_response_with_retry__partial_content_reponse(
self, mock_get_thread_session):
mock_requests_response = mock.Mock(status_code=206)
mock_requests_session = mock.create_autospec(requests.Session)
mock_requests_session.get.return_value = mock_requests_response
mock_get_thread_session.return_value = mock_requests_session
mock_presigned_url_provider = mock.create_autospec(
download_threads.PresignedUrlProvider)
presigned_url_info = download_threads.PresignedUrlInfo(
"foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow())
mock_presigned_url_provider.get_info.return_value = presigned_url_info
start = 5
end = 42
downloader = _MultithreadedDownloader(mock.Mock(), mock.Mock(), 5)
assert (
(start,
mock_requests_response) == downloader._get_response_with_retry(
mock_presigned_url_provider, start, end))
mock_requests_session.get.assert_called_once_with(
presigned_url_info.url,
headers={"Range": "bytes=5-42"},
stream=True)
def test_get_response_with_retry__exceed_max_retries(
self, mock_get_thread_session):
mock_requests_response = mock.Mock(status_code=403)
mock_requests_session = mock.create_autospec(requests.Session)
mock_requests_session.get.return_value = mock_requests_response
mock_get_thread_session.return_value = mock_requests_session
mock_presigned_url_provider = mock.create_autospec(
download_threads.PresignedUrlProvider)
presigned_url_info = download_threads.PresignedUrlInfo(
"foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow())
mock_presigned_url_provider.get_info.return_value = presigned_url_info
start = 5
end = 42
downloader = _MultithreadedDownloader(mock.Mock(), mock.Mock(), 5)
with pytest.raises(SynapseError):
downloader._get_response_with_retry(mock_presigned_url_provider,
start, end)
expected_call_list = [
mock.call(presigned_url_info.url,
headers={"Range": "bytes=5-42"},
stream=True)
] * download_threads.MAX_RETRIES
assert expected_call_list == mock_requests_session.get.call_args_list
def test_get_pre_signed_info(self):
fake_exp_time = datetime.datetime.utcnow()
fake_url = "https://synapse.org/foo.txt"
fake_file_name = "foo.txt"
with mock.patch.object(download_threads,
"_pre_signed_url_expiration_time",
return_value=fake_exp_time
) as mock_pre_signed_url_expiration_time:
fake_file_handle_response = {
"fileHandle": {
"fileName": fake_file_name
},
"preSignedURL": fake_url
}
self.mock_synapse_client._getFileHandleDownload.return_value = fake_file_handle_response
presigned_url_provider = download_threads.PresignedUrlProvider(
self.mock_synapse_client, self.download_request)
expected = download_threads.PresignedUrlInfo(
fake_file_name, fake_url, fake_exp_time)
assert_equals(expected,
presigned_url_provider._get_pre_signed_info())
mock_pre_signed_url_expiration_time.assert_called_with(fake_url)
def test_get_response_with_retry__error_status(self,
mock_get_thread_session):
"""Verify an errored status code during a part download will be retried"""
mock_requests_error_response = mock.Mock(status_code=500)
mock_requests_response = mock.Mock(status_code=206)
mock_requests_session = mock.create_autospec(requests.Session)
mock_requests_session.get.side_effect = [
mock_requests_error_response,
mock_requests_response,
]
mock_get_thread_session.return_value = mock_requests_session
mock_presigned_url_provider = mock.create_autospec(
download_threads.PresignedUrlProvider)
presigned_url_info = download_threads.PresignedUrlInfo(
"foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow())
mock_presigned_url_provider.get_info.return_value = presigned_url_info
start = 5
end = 42
mock_syn = mock.Mock(spec=Synapse)
mock_executor = mock.Mock(spec=concurrent.futures.Executor)
downloader = _MultithreadedDownloader(mock_syn, mock_executor, 5)
assert (
(start,
mock_requests_response) == downloader._get_response_with_retry(
mock_presigned_url_provider, start, end))
expected_get_call_args_list = [
mock.call(presigned_url_info.url, headers={"Range": "bytes=5-42"})
] * 2
assert mock_requests_session.get.call_args_list == expected_get_call_args_list
def test_get_info_not_expired(self):
utc_now = datetime.datetime.utcnow()
info = download_threads.PresignedUrlInfo(
"myFile.txt",
"https://synapse.org/somefile.txt",
expiration_utc=utc_now + datetime.timedelta(seconds=6))
with mock.patch.object(download_threads.PresignedUrlProvider, '_get_pre_signed_info',
return_value=info) as mock_get_presigned_info, \
mock.patch.object(download_threads, "datetime", wraps=datetime) as mock_datetime:
mock_datetime.datetime.utcnow.return_value = utc_now
presigned_url_provider = download_threads.PresignedUrlProvider(
self.mock_synapse_client, self.download_request)
assert_equals(info, presigned_url_provider.get_info())
# only caled once in init
mock_get_presigned_info.assert_called_once()
mock_datetime.datetime.utcnow.assert_called_once()
def setup(self):
self.mock_data_queue = mock.create_autospec(
download_threads.CloseableQueue)
self.mock_range_queue = mock.create_autospec(
download_threads.CloseableQueue)
self.mock_presigned_url_provider = mock.create_autospec(
download_threads.PresignedUrlProvider)
self.presigned_url_info = download_threads.PresignedUrlInfo(
"foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow())
self.mock_presigned_url_provider.get_info.return_value = self.presigned_url_info
self.mock_requests_session = mock.create_autospec(requests.Session)
self.mock_requests_response = mock.create_autospec(requests.Response)
self.mock_requests_session.get.return_value = self.mock_requests_response
self.response_bytes = [b'some bytes', b'some more bytes']
self.mock_requests_response.iter_content.return_value = self.response_bytes
response_byte_len = sum(len(x) for x in self.response_bytes)
self.mock_range_queue.__iter__.return_value = [
(0, response_byte_len - 1),
(response_byte_len, response_byte_len * 2)
]