def test_multiple_downloads(self):
cfg = config.DownloaderConfig()
lst = listener.AggregatingEventListener()
downloader = threaded.HTTPThreadedDownloader(cfg, lst)
bogus_file_names = ['notme', 'notmeeither']
all_file_names = self.data_file_names + bogus_file_names
url_list = [
'http://localhost:%d/%s%s' %
(self.server_port, self.data_directory, n) for n in all_file_names
]
dest_list = [
os.path.join(self.download_dir, n) for n in all_file_names
]
request_list = [
request.DownloadRequest(u, d) for u, d in zip(url_list, dest_list)
]
downloader.download(request_list)
self.assertEqual(len(lst.succeeded_reports), len(self.data_file_names))
self.assertEqual(len(lst.failed_reports), len(bogus_file_names))
for i, dest in enumerate(dest_list[:len(self.data_file_names)]):
self.assertTrue(os.path.exists(dest))
self.assertEqual(os.path.getsize(dest), self.data_file_sizes[i])
for dest in dest_list[len(self.data_file_names):]:
self.assertFalse(os.path.exists(dest))
def setUp(self):
self.config = config.DownloaderConfig()
self.listener = listener.AggregatingEventListener()
self.session = mock.Mock()
self.downloader = threaded.HTTPThreadedDownloader(self.config,
self.listener,
session=self.session)
def test_calls_fetch(self, mock_fetch):
config = DownloaderConfig()
request = DownloadRequest('http://foo', StringIO())
report = DownloadReport.from_download_request(request)
downloader = threaded.HTTPThreadedDownloader(config)
mock_fetch.return_value = report
ret = downloader._download_one(request)
self.assertEqual(mock_fetch.call_count, 1)
self.assertTrue(ret is report)
self.assertTrue(mock_fetch.call_args[0][0] is request)
def test_single_download_success(self):
cfg = config.DownloaderConfig()
lst = listener.AggregatingEventListener()
downloader = threaded.HTTPThreadedDownloader(cfg, lst)
file_path = os.path.join(self.data_directory, self.data_file_names[0])
dest_path = os.path.join(self.download_dir, self.data_file_names[0])
url = 'http://localhost:%d/%s' % (self.server_port, file_path)
req = request.DownloadRequest(url, dest_path)
downloader.download([req])
self.assertTrue(os.path.exists(dest_path))
self.assertEqual(os.path.getsize(dest_path), self.data_file_sizes[0])
self.assertEqual(len(lst.succeeded_reports), 1)
self.assertEqual(len(lst.failed_reports), 0)
def test_single_download_failure(self):
cfg = config.DownloaderConfig()
lst = listener.AggregatingEventListener()
downloader = threaded.HTTPThreadedDownloader(cfg, lst)
file_name = 'idontexistanddontcreateme'
file_path = os.path.join(self.data_directory, file_name)
dest_path = os.path.join(self.download_dir, file_name)
url = 'http://localhost:%d/%s' % (self.server_port, file_path)
req = request.DownloadRequest(url, dest_path)
downloader.download([req])
self.assertFalse(os.path.exists(dest_path))
self.assertEqual(len(lst.succeeded_reports), 0)
self.assertEqual(len(lst.failed_reports), 1)
self.assertTrue(lst.failed_reports[0].error_msg is not None)
def test_instantiation(self):
cfg = config.DownloaderConfig()
lst = listener.DownloadEventListener()
try:
downloader = threaded.HTTPThreadedDownloader(cfg, lst)
except:
self.fail('instantiation of requests eventlet downloader failed')
self.assertEqual(cfg, downloader.config)
self.assertEqual(lst, downloader.event_listener)
self.assertEqual(downloader.buffer_size, threaded.DEFAULT_BUFFER_SIZE)
self.assertEqual(downloader.max_concurrent,
threaded.DEFAULT_MAX_CONCURRENT)
self.assertEqual(
downloader.progress_interval,
datetime.timedelta(seconds=threaded.DEFAULT_PROGRESS_INTERVAL))
def test_download_unhandled_exception(self):
with mock.patch('nectar.downloaders.threaded._logger') as mock_logger:
cfg = config.DownloaderConfig()
lst = listener.AggregatingEventListener()
downloader = threaded.HTTPThreadedDownloader(cfg,
lst,
session=mock.Mock())
downloader._fetch = mock.Mock(side_effect=OSError)
downloader.download([mock.Mock()])
self.assertTrue(downloader.is_canceled)
expected_log_message = 'Unhandled Exception in Worker Thread'
log_calls = [
mock_call[1][0] for mock_call in mock_logger.mock_calls
]
self.assertIn(expected_log_message, log_calls[1])
def test_throttling(self):
two_seconds = datetime.timedelta(seconds=2)
three_seconds = datetime.timedelta(seconds=4)
cfg = config.DownloaderConfig(max_speed=256000) # 1/2 size of file
lst = listener.AggregatingEventListener()
downloader = threaded.HTTPThreadedDownloader(cfg, lst)
# use the 500k file, should take >= 2 seconds to download, but < 4
file_path = os.path.join(self.data_directory, self.data_file_names[1])
dest_path = os.path.join(self.download_dir, self.data_file_names[1])
url = 'http://localhost:%d/%s' % (self.server_port, file_path)
req = request.DownloadRequest(url, dest_path)
start = datetime.datetime.now()
downloader.download([req])
finish = datetime.datetime.now()
self.assertTrue(finish - start >= two_seconds)
self.assertTrue(finish - start < three_seconds)
