def test_multiple_downloads(self): cfg = config.DownloaderConfig() lst = listener.AggregatingEventListener() downloader = threaded.HTTPThreadedDownloader(cfg, lst) bogus_file_names = ['notme', 'notmeeither'] all_file_names = self.data_file_names + bogus_file_names url_list = [ 'http://localhost:%d/%s%s' % (self.server_port, self.data_directory, n) for n in all_file_names ] dest_list = [ os.path.join(self.download_dir, n) for n in all_file_names ] request_list = [ request.DownloadRequest(u, d) for u, d in zip(url_list, dest_list) ] downloader.download(request_list) self.assertEqual(len(lst.succeeded_reports), len(self.data_file_names)) self.assertEqual(len(lst.failed_reports), len(bogus_file_names)) for i, dest in enumerate(dest_list[:len(self.data_file_names)]): self.assertTrue(os.path.exists(dest)) self.assertEqual(os.path.getsize(dest), self.data_file_sizes[i]) for dest in dest_list[len(self.data_file_names):]: self.assertFalse(os.path.exists(dest))
def setUp(self): self.config = config.DownloaderConfig() self.listener = listener.AggregatingEventListener() self.session = mock.Mock() self.downloader = threaded.HTTPThreadedDownloader(self.config, self.listener, session=self.session)
def test_calls_fetch(self, mock_fetch): config = DownloaderConfig() request = DownloadRequest('http://foo', StringIO()) report = DownloadReport.from_download_request(request) downloader = threaded.HTTPThreadedDownloader(config) mock_fetch.return_value = report ret = downloader._download_one(request) self.assertEqual(mock_fetch.call_count, 1) self.assertTrue(ret is report) self.assertTrue(mock_fetch.call_args[0][0] is request)
def test_single_download_success(self): cfg = config.DownloaderConfig() lst = listener.AggregatingEventListener() downloader = threaded.HTTPThreadedDownloader(cfg, lst) file_path = os.path.join(self.data_directory, self.data_file_names[0]) dest_path = os.path.join(self.download_dir, self.data_file_names[0]) url = 'http://localhost:%d/%s' % (self.server_port, file_path) req = request.DownloadRequest(url, dest_path) downloader.download([req]) self.assertTrue(os.path.exists(dest_path)) self.assertEqual(os.path.getsize(dest_path), self.data_file_sizes[0]) self.assertEqual(len(lst.succeeded_reports), 1) self.assertEqual(len(lst.failed_reports), 0)
def test_single_download_failure(self): cfg = config.DownloaderConfig() lst = listener.AggregatingEventListener() downloader = threaded.HTTPThreadedDownloader(cfg, lst) file_name = 'idontexistanddontcreateme' file_path = os.path.join(self.data_directory, file_name) dest_path = os.path.join(self.download_dir, file_name) url = 'http://localhost:%d/%s' % (self.server_port, file_path) req = request.DownloadRequest(url, dest_path) downloader.download([req]) self.assertFalse(os.path.exists(dest_path)) self.assertEqual(len(lst.succeeded_reports), 0) self.assertEqual(len(lst.failed_reports), 1) self.assertTrue(lst.failed_reports[0].error_msg is not None)
def test_instantiation(self): cfg = config.DownloaderConfig() lst = listener.DownloadEventListener() try: downloader = threaded.HTTPThreadedDownloader(cfg, lst) except: self.fail('instantiation of requests eventlet downloader failed') self.assertEqual(cfg, downloader.config) self.assertEqual(lst, downloader.event_listener) self.assertEqual(downloader.buffer_size, threaded.DEFAULT_BUFFER_SIZE) self.assertEqual(downloader.max_concurrent, threaded.DEFAULT_MAX_CONCURRENT) self.assertEqual( downloader.progress_interval, datetime.timedelta(seconds=threaded.DEFAULT_PROGRESS_INTERVAL))
def test_download_unhandled_exception(self): with mock.patch('nectar.downloaders.threaded._logger') as mock_logger: cfg = config.DownloaderConfig() lst = listener.AggregatingEventListener() downloader = threaded.HTTPThreadedDownloader(cfg, lst, session=mock.Mock()) downloader._fetch = mock.Mock(side_effect=OSError) downloader.download([mock.Mock()]) self.assertTrue(downloader.is_canceled) expected_log_message = 'Unhandled Exception in Worker Thread' log_calls = [ mock_call[1][0] for mock_call in mock_logger.mock_calls ] self.assertIn(expected_log_message, log_calls[1])
def test_throttling(self): two_seconds = datetime.timedelta(seconds=2) three_seconds = datetime.timedelta(seconds=4) cfg = config.DownloaderConfig(max_speed=256000) # 1/2 size of file lst = listener.AggregatingEventListener() downloader = threaded.HTTPThreadedDownloader(cfg, lst) # use the 500k file, should take >= 2 seconds to download, but < 4 file_path = os.path.join(self.data_directory, self.data_file_names[1]) dest_path = os.path.join(self.download_dir, self.data_file_names[1]) url = 'http://localhost:%d/%s' % (self.server_port, file_path) req = request.DownloadRequest(url, dest_path) start = datetime.datetime.now() downloader.download([req]) finish = datetime.datetime.now() self.assertTrue(finish - start >= two_seconds) self.assertTrue(finish - start < three_seconds)