def test_get_provider_settings_no_data(self): """get_provider_settings() must return an empty dict if no matching entry is found""" download_manager = downloaders.DownloadManager( provider_settings_path=os.path.join(os.path.dirname(__file__), 'data/provider_settings.yml')) self.assertDictEqual( download_manager.get_provider_settings('https://foo.bar'), {})
def test_download_all_matched_datasets(self): """Test downloading all datasets matching the criteria""" download_manager = downloaders.DownloadManager( source__instrument__short_name='SLSTR') with mock.patch.object(downloaders.DownloadManager, 'download_dataset') as mock_dl_dataset: # Append the primary key to the results list instead of actually downloading mock_dl_dataset.side_effect = lambda d, _: d.pk self.assertListEqual(download_manager.download(), [2, 3])
def test_download_dataset_without_local_link_fails(self): """Test that `download_dataset` raises a DownloadError exception if the download failed""" download_manager = downloaders.DownloadManager() dataset = Dataset.objects.get(pk=1) with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: mock_dl_url.side_effect = downloaders.DownloadError with self.assertRaises(downloaders.DownloadError): with self.assertLogs(downloaders.LOGGER, logging.WARNING): download_manager.download_dataset(dataset, '')
def test_retrieve_datasets(self): """ Test that datasets are correctly retrieved according to the criteria given in the constructor """ download_manager = downloaders.DownloadManager( source__instrument__short_name='SLSTR') self.assertListEqual( list(download_manager.datasets), [Dataset.objects.get(pk=2), Dataset.objects.get(pk=3)])
def test_download_dataset_locked(self): """Test that an exception is raised if the max number of downloads has been reached""" download_manager = downloaders.DownloadManager() dataset = Dataset.objects.get(pk=1) with mock.patch.object(downloaders.DownloadLock, '__enter__') as mock_lock: with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: mock_lock.return_value = False with self.assertRaises(downloaders.TooManyDownloadsError): download_manager.download_dataset(dataset, '') mock_dl_url.assert_not_called()
def test_get_provider_settings(self): """Test that the settings for a particular provider are correctly retrieved""" download_manager = downloaders.DownloadManager( provider_settings_path=os.path.join(os.path.dirname(__file__), 'data/provider_settings.yml')) self.assertDictEqual( download_manager.get_provider_settings( 'https://scihub.copernicus.eu'), { 'username': '******', 'password': '******', 'max_parallel_downloads': 2 })
def test_download_no_downloader_found(self): """Test that `download_dataset` raises an exception when no downloader is found""" download_manager = downloaders.DownloadManager() download_manager.DOWNLOADERS = {} dataset = Dataset.objects.get(pk=1) with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: with self.assertLogs(downloaders.LOGGER): with self.assertRaises(KeyError): download_manager.download_dataset(dataset, '') mock_dl_url.assert_not_called()
def main(): """ Instantiation and calling the download() method of DownloadManager based on created argparser. """ arg = cli_parse_args() cumulative_query = util.create_cumulative_query(arg) download_manager = downloaders.DownloadManager( download_directory=arg.destination_path.rstrip(os.path.sep), provider_settings_path=arg.config_file, max_downloads=int(arg.safety_limit), save_path=arg.save_path, **cumulative_query) download_manager.download()
def test_trigger_download_if_no_max_downloads_settings_found(self): """ The download must be triggered if no max_parallel_downloads property exists for the provider """ download_manager = downloaders.DownloadManager() with mock.patch.object(downloaders.DownloadManager, 'get_provider_settings') as mock_p_s: mock_p_s.return_value = {} with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: mock_dl_url.return_value = 'Dataset_1_test.nc' download_manager.download_dataset(Dataset.objects.get(pk=1), '') mock_dl_url.assert_called()
def test_download_dataset_file_not_found_error(self): """ download_dataset() must raise a DownloadError if a FileNotFoundError or IsADirectoryError occurs when writing the downloaded file """ download_manager = downloaders.DownloadManager() with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: for error in [FileNotFoundError, IsADirectoryError]: mock_dl_url.side_effect = error with self.assertRaises(downloaders.DownloadError): with self.assertLogs(downloaders.LOGGER): download_manager.download_dataset( Dataset.objects.get(pk=1), '')
def test_save_path_if_file_already_exists(self): """ Test that address of previously downloaded file is added to the dataseturi model. in the case of lack of downloading action. """ download_manager = downloaders.DownloadManager(save_path=True) dataset = Dataset.objects.get(pk=3) with mock.patch('os.path.isdir', return_value=True), \ mock.patch('os.listdir', return_value=['test.nc']): download_manager.download_dataset(dataset, '/testing_value') self.assertEqual( dataset.dataseturi_set.filter( dataset=dataset, service=LOCAL_FILE_SERVICE)[0].uri, os.path.join('/testing_value', dataset.entry_id, 'test.nc'))
def test_load_provider_settings(self): """Test that provider settings are correctly loaded""" download_manager = downloaders.DownloadManager( provider_settings_path=os.path.join(os.path.dirname(__file__), 'data/provider_settings.yml')) self.assertDictEqual( download_manager.provider_settings, { 'https://scihub.copernicus.eu': { 'username': '******', 'password': '******', 'max_parallel_downloads': 2 }, 'https://random.url': { 'max_parallel_downloads': 10 } })
def test_save_path(self): """ Test that address of downloaded file is added to the dataseturi model in the case of downloading file. """ download_manager = downloaders.DownloadManager(save_path=True) dataset = Dataset.objects.get(pk=3) with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url', return_value='test.nc'), \ mock.patch('os.makedirs'): download_manager.download_dataset(dataset, '/testing_value') self.assertEqual( dataset.dataseturi_set.filter( dataset=dataset, service=LOCAL_FILE_SERVICE)[0].uri, os.path.join('/testing_value', dataset.entry_id, 'test.nc'))
def test_download_dataset_file_exists(self): """ Test that if the dataset file already exists, the existing file's path is returned and a debug message is logged. """ download_manager = downloaders.DownloadManager() dataset = Dataset.objects.get(pk=1) with mock.patch('os.path.isdir', return_value=True), \ mock.patch('os.listdir', return_value=['dataset_1_file.h5']): with self.assertLogs(logger=downloaders.LOGGER, level=logging.DEBUG) as logs_cm: result = download_manager.download_dataset( dataset, 'test_folder') self.assertTrue( "is already present at" in logs_cm.records[0].message) self.assertEqual( result, os.path.join(dataset.entry_id, 'dataset_1_file.h5'))
def test_download_dataset(self): """Test that a dataset is downloaded with the correct arguments""" download_manager = downloaders.DownloadManager( provider_settings_path=os.path.join(os.path.dirname(__file__), 'data/provider_settings.yml')) dataset = Dataset.objects.get(pk=1) dataset_url = dataset.dataseturi_set.first().uri with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: mock_dl_url.return_value = 'dataset_1_file.h5' result = download_manager.download_dataset(dataset, '') mock_dl_url.assert_called_with(url=dataset_url, download_dir=dataset.entry_id, username='******', password='******', max_parallel_downloads=2) self.assertEqual( result, os.path.join(dataset.entry_id, 'dataset_1_file.h5'))
def test_download_dataset_from_second_url(self): """Test downloading a dataset using its second URL if the first one fails""" download_manager = downloaders.DownloadManager() dataset = Dataset.objects.get(pk=1) dataset_file_name = 'dataset_1_file' # Function used to mock a download failure on the first URL def check_and_download_url_side_effect(url, download_dir, **kwargs): # pylint: disable=unused-argument if url == 'https://scihub.copernicus.eu/fakeurl': return dataset_file_name else: raise downloaders.DownloadError() with mock.patch.object(downloaders.HTTPDownloader, 'check_and_download_url') as mock_dl_url: mock_dl_url.side_effect = check_and_download_url_side_effect with self.assertLogs(logger=downloaders.LOGGER, level=logging.WARNING) as logs_cm: self.assertEqual( download_manager.download_dataset(dataset, ''), os.path.join(dataset.entry_id, dataset_file_name)) self.assertTrue(logs_cm.records[0].message.startswith( 'Failed to download dataset'))
def test_error_if_no_datasets_found(self): """An error must be raised if no dataset matches the criteria""" with self.assertRaises(downloaders.DownloadError): downloaders.DownloadManager(pk=100)
def test_error_on_too_wide_criteria(self): """Test that the download manager raises an error when too many datasets are found""" with self.assertRaises(ValueError): downloaders.DownloadManager(max_downloads=1, source__instrument__short_name='SLSTR')