def test_get_provider_settings_no_data(self):
"""get_provider_settings() must return an empty dict if no matching entry is found"""
download_manager = downloaders.DownloadManager(
provider_settings_path=os.path.join(os.path.dirname(__file__),
'data/provider_settings.yml'))
self.assertDictEqual(
download_manager.get_provider_settings('https://foo.bar'), {})
def test_download_all_matched_datasets(self):
"""Test downloading all datasets matching the criteria"""
download_manager = downloaders.DownloadManager(
source__instrument__short_name='SLSTR')
with mock.patch.object(downloaders.DownloadManager,
'download_dataset') as mock_dl_dataset:
# Append the primary key to the results list instead of actually downloading
mock_dl_dataset.side_effect = lambda d, _: d.pk
self.assertListEqual(download_manager.download(), [2, 3])
def test_download_dataset_without_local_link_fails(self):
"""Test that `download_dataset` raises a DownloadError exception if the download failed"""
download_manager = downloaders.DownloadManager()
dataset = Dataset.objects.get(pk=1)
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
mock_dl_url.side_effect = downloaders.DownloadError
with self.assertRaises(downloaders.DownloadError):
with self.assertLogs(downloaders.LOGGER, logging.WARNING):
download_manager.download_dataset(dataset, '')
def test_retrieve_datasets(self):
"""
Test that datasets are correctly retrieved according to the criteria given in the
constructor
"""
download_manager = downloaders.DownloadManager(
source__instrument__short_name='SLSTR')
self.assertListEqual(
list(download_manager.datasets),
[Dataset.objects.get(pk=2),
Dataset.objects.get(pk=3)])
def test_download_dataset_locked(self):
"""Test that an exception is raised if the max number of downloads has been reached"""
download_manager = downloaders.DownloadManager()
dataset = Dataset.objects.get(pk=1)
with mock.patch.object(downloaders.DownloadLock,
'__enter__') as mock_lock:
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
mock_lock.return_value = False
with self.assertRaises(downloaders.TooManyDownloadsError):
download_manager.download_dataset(dataset, '')
mock_dl_url.assert_not_called()
def test_get_provider_settings(self):
"""Test that the settings for a particular provider are correctly retrieved"""
download_manager = downloaders.DownloadManager(
provider_settings_path=os.path.join(os.path.dirname(__file__),
'data/provider_settings.yml'))
self.assertDictEqual(
download_manager.get_provider_settings(
'https://scihub.copernicus.eu'), {
'username': '******',
'password': '******',
'max_parallel_downloads': 2
})
def test_download_no_downloader_found(self):
"""Test that `download_dataset` raises an exception when no downloader is found"""
download_manager = downloaders.DownloadManager()
download_manager.DOWNLOADERS = {}
dataset = Dataset.objects.get(pk=1)
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
with self.assertLogs(downloaders.LOGGER):
with self.assertRaises(KeyError):
download_manager.download_dataset(dataset, '')
mock_dl_url.assert_not_called()
def main():
"""
Instantiation and calling the download() method of DownloadManager based on created argparser.
"""
arg = cli_parse_args()
cumulative_query = util.create_cumulative_query(arg)
download_manager = downloaders.DownloadManager(
download_directory=arg.destination_path.rstrip(os.path.sep),
provider_settings_path=arg.config_file,
max_downloads=int(arg.safety_limit),
save_path=arg.save_path,
**cumulative_query)
download_manager.download()
def test_trigger_download_if_no_max_downloads_settings_found(self):
"""
The download must be triggered if no max_parallel_downloads property exists for the provider
"""
download_manager = downloaders.DownloadManager()
with mock.patch.object(downloaders.DownloadManager,
'get_provider_settings') as mock_p_s:
mock_p_s.return_value = {}
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
mock_dl_url.return_value = 'Dataset_1_test.nc'
download_manager.download_dataset(Dataset.objects.get(pk=1),
'')
mock_dl_url.assert_called()
def test_download_dataset_file_not_found_error(self):
"""
download_dataset() must raise a DownloadError if a FileNotFoundError
or IsADirectoryError occurs when writing the downloaded file
"""
download_manager = downloaders.DownloadManager()
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
for error in [FileNotFoundError, IsADirectoryError]:
mock_dl_url.side_effect = error
with self.assertRaises(downloaders.DownloadError):
with self.assertLogs(downloaders.LOGGER):
download_manager.download_dataset(
Dataset.objects.get(pk=1), '')
def test_save_path_if_file_already_exists(self):
"""
Test that address of previously downloaded file is added to the dataseturi model. in the
case of lack of downloading action.
"""
download_manager = downloaders.DownloadManager(save_path=True)
dataset = Dataset.objects.get(pk=3)
with mock.patch('os.path.isdir', return_value=True), \
mock.patch('os.listdir', return_value=['test.nc']):
download_manager.download_dataset(dataset, '/testing_value')
self.assertEqual(
dataset.dataseturi_set.filter(
dataset=dataset, service=LOCAL_FILE_SERVICE)[0].uri,
os.path.join('/testing_value', dataset.entry_id, 'test.nc'))
def test_load_provider_settings(self):
"""Test that provider settings are correctly loaded"""
download_manager = downloaders.DownloadManager(
provider_settings_path=os.path.join(os.path.dirname(__file__),
'data/provider_settings.yml'))
self.assertDictEqual(
download_manager.provider_settings, {
'https://scihub.copernicus.eu': {
'username': '******',
'password': '******',
'max_parallel_downloads': 2
},
'https://random.url': {
'max_parallel_downloads': 10
}
})
def test_save_path(self):
"""
Test that address of downloaded file is added to the dataseturi model
in the case of downloading file.
"""
download_manager = downloaders.DownloadManager(save_path=True)
dataset = Dataset.objects.get(pk=3)
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url',
return_value='test.nc'), \
mock.patch('os.makedirs'):
download_manager.download_dataset(dataset, '/testing_value')
self.assertEqual(
dataset.dataseturi_set.filter(
dataset=dataset, service=LOCAL_FILE_SERVICE)[0].uri,
os.path.join('/testing_value', dataset.entry_id, 'test.nc'))
def test_download_dataset_file_exists(self):
"""
Test that if the dataset file already exists, the existing
file's path is returned and a debug message is logged.
"""
download_manager = downloaders.DownloadManager()
dataset = Dataset.objects.get(pk=1)
with mock.patch('os.path.isdir', return_value=True), \
mock.patch('os.listdir', return_value=['dataset_1_file.h5']):
with self.assertLogs(logger=downloaders.LOGGER,
level=logging.DEBUG) as logs_cm:
result = download_manager.download_dataset(
dataset, 'test_folder')
self.assertTrue(
"is already present at" in logs_cm.records[0].message)
self.assertEqual(
result, os.path.join(dataset.entry_id, 'dataset_1_file.h5'))
def test_download_dataset(self):
"""Test that a dataset is downloaded with the correct arguments"""
download_manager = downloaders.DownloadManager(
provider_settings_path=os.path.join(os.path.dirname(__file__),
'data/provider_settings.yml'))
dataset = Dataset.objects.get(pk=1)
dataset_url = dataset.dataseturi_set.first().uri
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
mock_dl_url.return_value = 'dataset_1_file.h5'
result = download_manager.download_dataset(dataset, '')
mock_dl_url.assert_called_with(url=dataset_url,
download_dir=dataset.entry_id,
username='******',
password='******',
max_parallel_downloads=2)
self.assertEqual(
result, os.path.join(dataset.entry_id, 'dataset_1_file.h5'))
def test_download_dataset_from_second_url(self):
"""Test downloading a dataset using its second URL if the first one fails"""
download_manager = downloaders.DownloadManager()
dataset = Dataset.objects.get(pk=1)
dataset_file_name = 'dataset_1_file'
# Function used to mock a download failure on the first URL
def check_and_download_url_side_effect(url, download_dir, **kwargs): # pylint: disable=unused-argument
if url == 'https://scihub.copernicus.eu/fakeurl':
return dataset_file_name
else:
raise downloaders.DownloadError()
with mock.patch.object(downloaders.HTTPDownloader,
'check_and_download_url') as mock_dl_url:
mock_dl_url.side_effect = check_and_download_url_side_effect
with self.assertLogs(logger=downloaders.LOGGER,
level=logging.WARNING) as logs_cm:
self.assertEqual(
download_manager.download_dataset(dataset, ''),
os.path.join(dataset.entry_id, dataset_file_name))
self.assertTrue(logs_cm.records[0].message.startswith(
'Failed to download dataset'))
def test_error_if_no_datasets_found(self):
"""An error must be raised if no dataset matches the criteria"""
with self.assertRaises(downloaders.DownloadError):
downloaders.DownloadManager(pk=100)
def test_error_on_too_wide_criteria(self):
"""Test that the download manager raises an error when too many datasets are found"""
with self.assertRaises(ValueError):
downloaders.DownloadManager(max_downloads=1,
source__instrument__short_name='SLSTR')