def test_get_info_expired(self): utc_now = datetime.datetime.utcnow() # expires in the past expired_info = download_threads.PresignedUrlInfo( "myFile.txt", "https://synapse.org/somefile.txt", expiration_utc=utc_now - datetime.timedelta(seconds=5)) unexpired_info = download_threads.PresignedUrlInfo( "myFile.txt", "https://synapse.org/somefile.txt", expiration_utc=utc_now + datetime.timedelta(seconds=6)) with mock.patch.object(download_threads.PresignedUrlProvider, '_get_pre_signed_info', side_effect=[expired_info, unexpired_info]) as mock_get_presigned_info, \ mock.patch.object(download_threads, "datetime") as mock_datetime: mock_datetime.datetime.utcnow.return_value = utc_now presigned_url_provider = download_threads.PresignedUrlProvider( self.mock_synapse_client, self.download_request) assert_equals(unexpired_info, presigned_url_provider.get_info()) # only caled once in init and again in get_info assert_equals(2, mock_get_presigned_info.call_count) mock_datetime.datetime.utcnow.assert_called_once()
def test_get_response_with_retry__partial_content_reponse( self, mock_get_thread_session): mock_requests_response = mock.Mock(status_code=206) mock_requests_session = mock.create_autospec(requests.Session) mock_requests_session.get.return_value = mock_requests_response mock_get_thread_session.return_value = mock_requests_session mock_presigned_url_provider = mock.create_autospec( download_threads.PresignedUrlProvider) presigned_url_info = download_threads.PresignedUrlInfo( "foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow()) mock_presigned_url_provider.get_info.return_value = presigned_url_info start = 5 end = 42 downloader = _MultithreadedDownloader(mock.Mock(), mock.Mock(), 5) assert ( (start, mock_requests_response) == downloader._get_response_with_retry( mock_presigned_url_provider, start, end)) mock_requests_session.get.assert_called_once_with( presigned_url_info.url, headers={"Range": "bytes=5-42"}, stream=True)
def test_get_response_with_retry__exceed_max_retries( self, mock_get_thread_session): mock_requests_response = mock.Mock(status_code=403) mock_requests_session = mock.create_autospec(requests.Session) mock_requests_session.get.return_value = mock_requests_response mock_get_thread_session.return_value = mock_requests_session mock_presigned_url_provider = mock.create_autospec( download_threads.PresignedUrlProvider) presigned_url_info = download_threads.PresignedUrlInfo( "foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow()) mock_presigned_url_provider.get_info.return_value = presigned_url_info start = 5 end = 42 downloader = _MultithreadedDownloader(mock.Mock(), mock.Mock(), 5) with pytest.raises(SynapseError): downloader._get_response_with_retry(mock_presigned_url_provider, start, end) expected_call_list = [ mock.call(presigned_url_info.url, headers={"Range": "bytes=5-42"}, stream=True) ] * download_threads.MAX_RETRIES assert expected_call_list == mock_requests_session.get.call_args_list
def test_get_pre_signed_info(self): fake_exp_time = datetime.datetime.utcnow() fake_url = "https://synapse.org/foo.txt" fake_file_name = "foo.txt" with mock.patch.object(download_threads, "_pre_signed_url_expiration_time", return_value=fake_exp_time ) as mock_pre_signed_url_expiration_time: fake_file_handle_response = { "fileHandle": { "fileName": fake_file_name }, "preSignedURL": fake_url } self.mock_synapse_client._getFileHandleDownload.return_value = fake_file_handle_response presigned_url_provider = download_threads.PresignedUrlProvider( self.mock_synapse_client, self.download_request) expected = download_threads.PresignedUrlInfo( fake_file_name, fake_url, fake_exp_time) assert_equals(expected, presigned_url_provider._get_pre_signed_info()) mock_pre_signed_url_expiration_time.assert_called_with(fake_url)
def test_get_response_with_retry__error_status(self, mock_get_thread_session): """Verify an errored status code during a part download will be retried""" mock_requests_error_response = mock.Mock(status_code=500) mock_requests_response = mock.Mock(status_code=206) mock_requests_session = mock.create_autospec(requests.Session) mock_requests_session.get.side_effect = [ mock_requests_error_response, mock_requests_response, ] mock_get_thread_session.return_value = mock_requests_session mock_presigned_url_provider = mock.create_autospec( download_threads.PresignedUrlProvider) presigned_url_info = download_threads.PresignedUrlInfo( "foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow()) mock_presigned_url_provider.get_info.return_value = presigned_url_info start = 5 end = 42 mock_syn = mock.Mock(spec=Synapse) mock_executor = mock.Mock(spec=concurrent.futures.Executor) downloader = _MultithreadedDownloader(mock_syn, mock_executor, 5) assert ( (start, mock_requests_response) == downloader._get_response_with_retry( mock_presigned_url_provider, start, end)) expected_get_call_args_list = [ mock.call(presigned_url_info.url, headers={"Range": "bytes=5-42"}) ] * 2 assert mock_requests_session.get.call_args_list == expected_get_call_args_list
def test_get_info_not_expired(self): utc_now = datetime.datetime.utcnow() info = download_threads.PresignedUrlInfo( "myFile.txt", "https://synapse.org/somefile.txt", expiration_utc=utc_now + datetime.timedelta(seconds=6)) with mock.patch.object(download_threads.PresignedUrlProvider, '_get_pre_signed_info', return_value=info) as mock_get_presigned_info, \ mock.patch.object(download_threads, "datetime", wraps=datetime) as mock_datetime: mock_datetime.datetime.utcnow.return_value = utc_now presigned_url_provider = download_threads.PresignedUrlProvider( self.mock_synapse_client, self.download_request) assert_equals(info, presigned_url_provider.get_info()) # only caled once in init mock_get_presigned_info.assert_called_once() mock_datetime.datetime.utcnow.assert_called_once()
def setup(self): self.mock_data_queue = mock.create_autospec( download_threads.CloseableQueue) self.mock_range_queue = mock.create_autospec( download_threads.CloseableQueue) self.mock_presigned_url_provider = mock.create_autospec( download_threads.PresignedUrlProvider) self.presigned_url_info = download_threads.PresignedUrlInfo( "foo.txt", "synapse.org/foo.txt", datetime.datetime.utcnow()) self.mock_presigned_url_provider.get_info.return_value = self.presigned_url_info self.mock_requests_session = mock.create_autospec(requests.Session) self.mock_requests_response = mock.create_autospec(requests.Response) self.mock_requests_session.get.return_value = self.mock_requests_response self.response_bytes = [b'some bytes', b'some more bytes'] self.mock_requests_response.iter_content.return_value = self.response_bytes response_byte_len = sum(len(x) for x in self.response_bytes) self.mock_range_queue.__iter__.return_value = [ (0, response_byte_len - 1), (response_byte_len, response_byte_len * 2) ]