def upload_output(storage_provider, output_dir_path):
"""Receives the tmp_dir_path where the files to upload are stored and
uploads all the files found there."""
get_logger().info("Searching for files to upload in folder '%s'", output_dir_path)
output_files = FileUtils.get_all_files_in_dir(output_dir_path)
get_logger().info("Found the following files to upload: '%s'", output_files)
for file_path in output_files:
file_name = file_path.replace(f"{output_dir_path}/", "")
storage_provider.upload_file(file_path, file_name)
def upload_output(self, output_dir_path):
"""Receives the tmp_dir_path where the files to upload are stored and
uploads files whose name matches the prefixes and suffixes specified
in 'output'."""
get_logger().info('Searching for files to upload in folder \'%s\'',
output_dir_path)
output_files = FileUtils.get_all_files_in_dir(output_dir_path)
stg_providers = {}
# Filter files by prefix and suffix
for output in self.output:
get_logger().info(
'Checking files for uploading to \'%s\' on path: \'%s\'',
output['storage_provider'], output['path'])
provider_type = StrUtils.get_storage_type(
output['storage_provider'])
provider_id = StrUtils.get_storage_id(output['storage_provider'])
for file_path in output_files:
file_name = file_path.replace(f'{output_dir_path}/', '')
prefix_ok = False
suffix_ok = False
# Check prefixes
if ('prefix' not in output or len(output['prefix']) == 0):
prefix_ok = True
else:
for pref in output['prefix']:
if file_name.startswith(pref):
prefix_ok = True
break
if prefix_ok:
# Check suffixes
if ('suffix' not in output or len(output['suffix']) == 0):
suffix_ok = True
else:
for suff in output['suffix']:
if file_name.endswith(suff):
suffix_ok = True
break
# Only upload file if name matches the prefixes and suffixes
if suffix_ok:
if provider_type not in stg_providers:
stg_providers[provider_type] = {}
if provider_id not in stg_providers[provider_type]:
auth_data = self._get_auth_data(
provider_type, provider_id)
stg_providers[provider_type][
provider_id] = create_provider(auth_data)
stg_providers[provider_type][provider_id].upload_file(
file_path, file_name, output['path'])
def test_get_all_files_in_dir(self, mock_os):
mock_os.return_value = [('/tmp', ['t1'], ['f1', 'f2']),
('/tmp/t1', [], ['f3'])]
files = FileUtils.get_all_files_in_dir('/tmp')
mock_os.assert_called_once_with('/tmp')
self.assertEqual(files, ['/tmp/f1', '/tmp/f2', '/tmp/t1/f3'])