def upload_output(self, output_dir_path):
"""Receives the tmp_dir_path where the files to upload are stored and
uploads files whose name matches the prefixes and suffixes specified
in 'output'."""
get_logger().info('Searching for files to upload in folder \'%s\'',
output_dir_path)
output_files = FileUtils.get_all_files_in_dir(output_dir_path)
stg_providers = {}
# Filter files by prefix and suffix
for output in self.output:
get_logger().info(
'Checking files for uploading to \'%s\' on path: \'%s\'',
output['storage_provider'], output['path'])
provider_type = StrUtils.get_storage_type(
output['storage_provider'])
provider_id = StrUtils.get_storage_id(output['storage_provider'])
for file_path in output_files:
file_name = file_path.replace(f'{output_dir_path}/', '')
prefix_ok = False
suffix_ok = False
# Check prefixes
if ('prefix' not in output or len(output['prefix']) == 0):
prefix_ok = True
else:
for pref in output['prefix']:
if file_name.startswith(pref):
prefix_ok = True
break
if prefix_ok:
# Check suffixes
if ('suffix' not in output or len(output['suffix']) == 0):
suffix_ok = True
else:
for suff in output['suffix']:
if file_name.endswith(suff):
suffix_ok = True
break
# Only upload file if name matches the prefixes and suffixes
if suffix_ok:
if provider_type not in stg_providers:
stg_providers[provider_type] = {}
if provider_id not in stg_providers[provider_type]:
auth_data = self._get_auth_data(
provider_type, provider_id)
stg_providers[provider_type][
provider_id] = create_provider(auth_data)
stg_providers[provider_type][provider_id].upload_file(
file_path, file_name, output['path'])
def _get_input_auth_data(self, parsed_event):
"""Return the proper auth data from a storage_provider based on the event.
This methods allows to filter ONEDATA provider when multiple inputs are defined."""
storage_type = parsed_event.get_type()
if storage_type == 'ONEDATA':
# Check input path and event object_key
if hasattr(parsed_event, 'object_key'):
# Get the onedata space from the event object_key
event_space = parsed_event.object_key.strip('/').split(
'/', maxsplit=1)[0]
for input in self.input:
provider_type = StrUtils.get_storage_type(
input.get('storage_provider'))
if provider_type == storage_type:
provider_id = StrUtils.get_storage_id(
input.get('storage_provider'))
if self.onedata_auth[provider_id].get_credential(
'space') == event_space:
return self._get_auth_data(storage_type, provider_id)
raise StorageAuthError(auth_type='ONEDATA')
else:
return self._get_auth_data(storage_type)
def test_get_storage_id(self):
self.assertEqual(StrUtils.get_storage_id('bad.good'), 'good')
self.assertEqual(StrUtils.get_storage_id('bad.good.with.dots'), 'good.with.dots')