def _get_map_uri_list(self):
"""
Get the contents of the map URI (agave URI).
Args:
self: class instance.
Returns:
Array of base file names in the map URI. Returns False on
exception.
"""
# make sure map URI is compatible scheme (agave)
if self._parsed_map_uri['scheme'] != 'agave':
msg = 'invalid map uri scheme for this step: {}'.format(
self._parsed_map_uri['scheme'])
Log.an().error(msg)
return self._fatal(msg)
# list files from URI
file_list = DataManager.list(parsed_uri=self._parsed_map_uri,
agave=self._agave)
if file_list is False:
msg = 'cannot get contents of map uri: {}'\
.format(self._parsed_map_uri['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
return file_list
def _init_app_paths(self):
"""
Add app paths to environment PATH for local workflows.
The package path contains the workflow definition YAML file and shell
scripts for calling individual apps used in a workflow.
Args:
None.
Output:
On success: True.
On failure: False.
"""
parsed_uri = URIParser.parse(self._workflow_path)
if not parsed_uri:
Log.an().error('invalid workflow path: %s', self._workflow_path)
return False
apps_uri = ('{}{}' if parsed_uri['folder'] == '/' else '{}/{}')\
.format(parsed_uri['folder'], 'apps')
parsed_apps_uri = URIParser.parse(
('{}{}' if parsed_uri['folder'] == '/' else '{}/{}')\
.format(parsed_uri['folder'], 'apps')
)
if not parsed_apps_uri:
Log.an().error('cannot construct apps uri: %s', apps_uri)
return False
if not DataManager.exists(parsed_uri=parsed_apps_uri):
# no apps directory
return True
for app_dir in DataManager.list(parsed_uri=parsed_apps_uri):
try:
os.environ['PATH'] = '{}{}{}'.format(
os.path.join(parsed_apps_uri['chopped_path'], app_dir,
'assets'), os.pathsep, os.environ['PATH'])
except OSError as err:
Log.an().error('workflow app pathmunge error [%s]', str(err))
return False
return True
def _get_map_uri_list(self):
"""
Get the contents of the map URI (agave URI).
Args:
self: class instance.
Returns:
Array of base file names in the map URI. Returns False on
exception.
"""
combined_file_list = []
for uri in self._parsed_map_uris:
# make sure map URI is compatible scheme (agave)
if uri['scheme'] != 'agave':
msg = 'invalid map uri scheme for this step: {}'.format(
uri['scheme']
)
Log.an().error(msg)
return self._fatal(msg)
# get file list from URI
file_list = DataManager.list(
parsed_uri=uri,
globstr=self._step['map']['glob'],
agave=self._agave
)
if file_list is False:
msg = 'cannot get contents of map uri: {}'\
.format(uri['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
for f in file_list:
combined_file_list.append({
'chopped_uri': uri['chopped_uri'],
'filename': f
})
return combined_file_list
def clean_up(self):
"""
Copy data from Agave archive location to step output location (data URI).
Args:
self: class instance.
Returns:
On success: True.
On failure: False.
"""
# destination _log directory, common for all map items
dest_log_dir = '{}/{}'.format(
self._parsed_data_uris[self._source_context][0]\
['chopped_uri'],
'_log'
)
# copy data for each map item
for map_item in self._map:
# copy step output
if not self._agave['agave_wrapper'].files_import_from_agave(
self._parsed_data_uris[self._source_context][0]['authority'],
self._parsed_data_uris[self._source_context][0]\
['chopped_path'],
map_item['template']['output'],
'{}/{}'.format(
map_item['run'][map_item['attempt']]['archive_uri'],
map_item['template']['output']
)
):
msg = 'agave import failed for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# check for any agave log files (*.out and *.err files)
agave_log_list = DataManager.list(
uri=map_item['run'][map_item['attempt']]['archive_uri'],
agave=self._agave
)
if agave_log_list is False:
msg = 'cannot get agave log list for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# copy each agave log file, the pattern is gf-{}-{}-{}.out or .err
for item in agave_log_list:
if re.match('^gf-\d*-.*\.(out|err)$', item):
if not self._agave['agave_wrapper'].files_import_from_agave(
self._parsed_data_uris[self._source_context][0]\
['authority'],
'{}/{}'.format(
self._parsed_data_uris[self._source_context][0]\
['chopped_path'],
'_log'
),
item,
'{}/{}'.format(
map_item['run'][map_item['attempt']]\
['archive_uri'],
item
)
):
msg = 'cannot copy agave log item "{}"'.format(item)
Log.an().error(msg)
return self._fatal(msg)
# check if anything is in the _log directory
src_log_dir = '{}/{}'.format(
map_item['run'][map_item['attempt']]['archive_uri'],
'_log'
)
if DataManager.exists(
uri=src_log_dir,
agave=self._agave
):
# create dest _log dir if it doesn't exist
if not DataManager.exists(
uri=dest_log_dir,
agave=self._agave
):
if not DataManager.mkdir(
uri=dest_log_dir,
agave=self._agave
):
msg = 'cannot create _log directory for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# get list of all items in src_log_dir
log_list = DataManager.list(
uri=src_log_dir,
agave=self._agave
)
if log_list is False:
msg = 'cannot get _log list for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# copy each list item
for item in log_list:
if not self._agave['agave_wrapper'].files_import_from_agave(
self._parsed_data_uris[self._source_context][0]\
['authority'],
'{}/{}'.format(
self._parsed_data_uris[self._source_context][0]\
['chopped_path'],
'_log'
),
item,
'{}/{}/{}'.format(
map_item['run'][map_item['attempt']]\
['archive_uri'],
'_log',
item
)
):
msg = 'cannot copy log item "{}"'.format(item)
Log.an().error(msg)
return self._fatal(msg)
self._update_status_db('FINISHED', '')
return True
def _get_map_uri_list(self):
"""
Get the contents of the map URI (agave URI).
Args:
self: class instance.
Returns:
Array of base file names in the map URI. Returns False on
exception.
"""
combined_file_list = []
for uri in self._parsed_map_uris:
# make sure map URI is compatible scheme (agave)
if uri['scheme'] != 'agave':
msg = 'invalid map uri scheme for this step: {}'.format(
uri['scheme']
)
Log.an().error(msg)
return self._fatal(msg)
# get file list from URI
file_list = DataManager.list(
parsed_uri=uri,
globstr=self._step['map']['glob'],
agave=self._agave
)
if file_list is False:
msg = 'cannot get contents of map uri: {}'\
.format(uri['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
if self._step['map']['inclusive']:
# filter with glob
if glob.globfilter(
[uri['name']],
self._step['map']['glob'],
flags=glob.EXTGLOB|glob.GLOBSTAR
):
combined_file_list.append({
'chopped_uri': '{}://{}{}'.format(
uri['scheme'],
uri['authority'],
uri['folder']
),
'filename': uri['name']
})
for f in file_list:
if '/' in f:
# reparse uri to correctly represent recursive elements
new_uri = URIParser.parse('{}/{}'.format(uri['chopped_uri'], f))
combined_file_list.append({
'chopped_uri': '{}://{}{}'.format(
new_uri['scheme'],
new_uri['authority'],
new_uri['folder']
),
'filename': new_uri['name']
})
else:
combined_file_list.append({
'chopped_uri': uri['chopped_uri'],
'filename': f
})
return combined_file_list
def clean_up(self):
"""
Copy data from Agave archive location to step output location (data URI).
Args:
self: class instance.
Returns:
On success: True.
On failure: False.
"""
# destination _log directory, common for all map items
dest_log_dir = '{}/{}'.format(
self._parsed_data_uris[self._source_context]\
['chopped_uri'],
'_log'
)
# create instance of agave wrapper class for data import
agwrap = AgaveFilesImportDataFromAgave(
self._agave['agave'],
self._config['agave']
)
# copy data for each map item
for map_item in self._map:
# copy step output
if not agwrap.call(
self._parsed_data_uris[self._source_context]['authority'],
self._parsed_data_uris[self._source_context]\
['chopped_path'],
map_item['template']['output'],
'{}/{}'.format(
map_item['run'][map_item['attempt']]['archive_uri'],
map_item['template']['output']
)
):
msg = 'agave import failed for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# check if anything is in the _log directory
src_log_dir = '{}/{}'.format(
map_item['run'][map_item['attempt']]['archive_uri'],
'_log'
)
if DataManager.exists(
uri=src_log_dir,
agave={
'agave': self._agave['agave'],
'agave_config': self._config['agave']
}
):
# create dest _log dir if it doesn't exist
if not DataManager.exists(
uri=dest_log_dir,
agave={
'agave': self._agave['agave'],
'agave_config': self._config['agave']
}
):
if not DataManager.mkdir(
uri=dest_log_dir,
agave={
'agave': self._agave['agave'],
'agave_config': self._config['agave']
}
):
msg = 'cannot create _log directory for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# get list of all items in src_log_dir
log_list = DataManager.list(
uri=src_log_dir,
agave={
'agave': self._agave['agave'],
'agave_config': self._config['agave']
}
)
if not log_list:
msg = 'cannot get _log list for step "{}"'\
.format(self._step['name'])
Log.an().error(msg)
return self._fatal(msg)
# copy each list item
for item in log_list:
if not agwrap.call(
self._parsed_data_uris[self._source_context]\
['authority'],
'{}/{}'.format(
self._parsed_data_uris[self._source_context]\
['chopped_path'],
'_log'
),
item,
'{}/{}/{}'.format(
map_item['run'][map_item['attempt']]\
['archive_uri'],
'_log',
item
)
):
msg = 'cannot copy log item "{}"'.format(item)
Log.an().error(msg)
return self._fatal(msg)
self._update_status_db('FINISHED', '')
return True