def _get_map_uri_list(self): """ Get the contents of the map URI (agave URI). Args: self: class instance. Returns: Array of base file names in the map URI. Returns False on exception. """ # make sure map URI is compatible scheme (agave) if self._parsed_map_uri['scheme'] != 'agave': msg = 'invalid map uri scheme for this step: {}'.format( self._parsed_map_uri['scheme']) Log.an().error(msg) return self._fatal(msg) # list files from URI file_list = DataManager.list(parsed_uri=self._parsed_map_uri, agave=self._agave) if file_list is False: msg = 'cannot get contents of map uri: {}'\ .format(self._parsed_map_uri['chopped_uri']) Log.an().error(msg) return self._fatal(msg) return file_list
def _init_app_paths(self): """ Add app paths to environment PATH for local workflows. The package path contains the workflow definition YAML file and shell scripts for calling individual apps used in a workflow. Args: None. Output: On success: True. On failure: False. """ parsed_uri = URIParser.parse(self._workflow_path) if not parsed_uri: Log.an().error('invalid workflow path: %s', self._workflow_path) return False apps_uri = ('{}{}' if parsed_uri['folder'] == '/' else '{}/{}')\ .format(parsed_uri['folder'], 'apps') parsed_apps_uri = URIParser.parse( ('{}{}' if parsed_uri['folder'] == '/' else '{}/{}')\ .format(parsed_uri['folder'], 'apps') ) if not parsed_apps_uri: Log.an().error('cannot construct apps uri: %s', apps_uri) return False if not DataManager.exists(parsed_uri=parsed_apps_uri): # no apps directory return True for app_dir in DataManager.list(parsed_uri=parsed_apps_uri): try: os.environ['PATH'] = '{}{}{}'.format( os.path.join(parsed_apps_uri['chopped_path'], app_dir, 'assets'), os.pathsep, os.environ['PATH']) except OSError as err: Log.an().error('workflow app pathmunge error [%s]', str(err)) return False return True
def _get_map_uri_list(self): """ Get the contents of the map URI (agave URI). Args: self: class instance. Returns: Array of base file names in the map URI. Returns False on exception. """ combined_file_list = [] for uri in self._parsed_map_uris: # make sure map URI is compatible scheme (agave) if uri['scheme'] != 'agave': msg = 'invalid map uri scheme for this step: {}'.format( uri['scheme'] ) Log.an().error(msg) return self._fatal(msg) # get file list from URI file_list = DataManager.list( parsed_uri=uri, globstr=self._step['map']['glob'], agave=self._agave ) if file_list is False: msg = 'cannot get contents of map uri: {}'\ .format(uri['chopped_uri']) Log.an().error(msg) return self._fatal(msg) for f in file_list: combined_file_list.append({ 'chopped_uri': uri['chopped_uri'], 'filename': f }) return combined_file_list
def clean_up(self): """ Copy data from Agave archive location to step output location (data URI). Args: self: class instance. Returns: On success: True. On failure: False. """ # destination _log directory, common for all map items dest_log_dir = '{}/{}'.format( self._parsed_data_uris[self._source_context][0]\ ['chopped_uri'], '_log' ) # copy data for each map item for map_item in self._map: # copy step output if not self._agave['agave_wrapper'].files_import_from_agave( self._parsed_data_uris[self._source_context][0]['authority'], self._parsed_data_uris[self._source_context][0]\ ['chopped_path'], map_item['template']['output'], '{}/{}'.format( map_item['run'][map_item['attempt']]['archive_uri'], map_item['template']['output'] ) ): msg = 'agave import failed for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # check for any agave log files (*.out and *.err files) agave_log_list = DataManager.list( uri=map_item['run'][map_item['attempt']]['archive_uri'], agave=self._agave ) if agave_log_list is False: msg = 'cannot get agave log list for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # copy each agave log file, the pattern is gf-{}-{}-{}.out or .err for item in agave_log_list: if re.match('^gf-\d*-.*\.(out|err)$', item): if not self._agave['agave_wrapper'].files_import_from_agave( self._parsed_data_uris[self._source_context][0]\ ['authority'], '{}/{}'.format( self._parsed_data_uris[self._source_context][0]\ ['chopped_path'], '_log' ), item, '{}/{}'.format( map_item['run'][map_item['attempt']]\ ['archive_uri'], item ) ): msg = 'cannot copy agave log item "{}"'.format(item) Log.an().error(msg) return self._fatal(msg) # check if anything is in the _log directory src_log_dir = '{}/{}'.format( map_item['run'][map_item['attempt']]['archive_uri'], '_log' ) if DataManager.exists( uri=src_log_dir, agave=self._agave ): # create dest _log dir if it doesn't exist if not DataManager.exists( uri=dest_log_dir, agave=self._agave ): if not DataManager.mkdir( uri=dest_log_dir, agave=self._agave ): msg = 'cannot create _log directory for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # get list of all items in src_log_dir log_list = DataManager.list( uri=src_log_dir, agave=self._agave ) if log_list is False: msg = 'cannot get _log list for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # copy each list item for item in log_list: if not self._agave['agave_wrapper'].files_import_from_agave( self._parsed_data_uris[self._source_context][0]\ ['authority'], '{}/{}'.format( self._parsed_data_uris[self._source_context][0]\ ['chopped_path'], '_log' ), item, '{}/{}/{}'.format( map_item['run'][map_item['attempt']]\ ['archive_uri'], '_log', item ) ): msg = 'cannot copy log item "{}"'.format(item) Log.an().error(msg) return self._fatal(msg) self._update_status_db('FINISHED', '') return True
def _get_map_uri_list(self): """ Get the contents of the map URI (agave URI). Args: self: class instance. Returns: Array of base file names in the map URI. Returns False on exception. """ combined_file_list = [] for uri in self._parsed_map_uris: # make sure map URI is compatible scheme (agave) if uri['scheme'] != 'agave': msg = 'invalid map uri scheme for this step: {}'.format( uri['scheme'] ) Log.an().error(msg) return self._fatal(msg) # get file list from URI file_list = DataManager.list( parsed_uri=uri, globstr=self._step['map']['glob'], agave=self._agave ) if file_list is False: msg = 'cannot get contents of map uri: {}'\ .format(uri['chopped_uri']) Log.an().error(msg) return self._fatal(msg) if self._step['map']['inclusive']: # filter with glob if glob.globfilter( [uri['name']], self._step['map']['glob'], flags=glob.EXTGLOB|glob.GLOBSTAR ): combined_file_list.append({ 'chopped_uri': '{}://{}{}'.format( uri['scheme'], uri['authority'], uri['folder'] ), 'filename': uri['name'] }) for f in file_list: if '/' in f: # reparse uri to correctly represent recursive elements new_uri = URIParser.parse('{}/{}'.format(uri['chopped_uri'], f)) combined_file_list.append({ 'chopped_uri': '{}://{}{}'.format( new_uri['scheme'], new_uri['authority'], new_uri['folder'] ), 'filename': new_uri['name'] }) else: combined_file_list.append({ 'chopped_uri': uri['chopped_uri'], 'filename': f }) return combined_file_list
def clean_up(self): """ Copy data from Agave archive location to step output location (data URI). Args: self: class instance. Returns: On success: True. On failure: False. """ # destination _log directory, common for all map items dest_log_dir = '{}/{}'.format( self._parsed_data_uris[self._source_context]\ ['chopped_uri'], '_log' ) # create instance of agave wrapper class for data import agwrap = AgaveFilesImportDataFromAgave( self._agave['agave'], self._config['agave'] ) # copy data for each map item for map_item in self._map: # copy step output if not agwrap.call( self._parsed_data_uris[self._source_context]['authority'], self._parsed_data_uris[self._source_context]\ ['chopped_path'], map_item['template']['output'], '{}/{}'.format( map_item['run'][map_item['attempt']]['archive_uri'], map_item['template']['output'] ) ): msg = 'agave import failed for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # check if anything is in the _log directory src_log_dir = '{}/{}'.format( map_item['run'][map_item['attempt']]['archive_uri'], '_log' ) if DataManager.exists( uri=src_log_dir, agave={ 'agave': self._agave['agave'], 'agave_config': self._config['agave'] } ): # create dest _log dir if it doesn't exist if not DataManager.exists( uri=dest_log_dir, agave={ 'agave': self._agave['agave'], 'agave_config': self._config['agave'] } ): if not DataManager.mkdir( uri=dest_log_dir, agave={ 'agave': self._agave['agave'], 'agave_config': self._config['agave'] } ): msg = 'cannot create _log directory for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # get list of all items in src_log_dir log_list = DataManager.list( uri=src_log_dir, agave={ 'agave': self._agave['agave'], 'agave_config': self._config['agave'] } ) if not log_list: msg = 'cannot get _log list for step "{}"'\ .format(self._step['name']) Log.an().error(msg) return self._fatal(msg) # copy each list item for item in log_list: if not agwrap.call( self._parsed_data_uris[self._source_context]\ ['authority'], '{}/{}'.format( self._parsed_data_uris[self._source_context]\ ['chopped_path'], '_log' ), item, '{}/{}/{}'.format( map_item['run'][map_item['attempt']]\ ['archive_uri'], '_log', item ) ): msg = 'cannot copy log item "{}"'.format(item) Log.an().error(msg) return self._fatal(msg) self._update_status_db('FINISHED', '') return True