def _init_data_uri(self): """ Create output data URI for the source context (local). Args: self: class instance. Returns: On success: True. On failure: False. """ # make sure the source data URI has a compatible scheme (local) if self._parsed_data_uris[self._source_context]['scheme'] != 'local': msg = 'invalid data uri scheme for this step: {}'.format( self._parsed_data_uris[self._source_context]['scheme'] ) Log.an().error(msg) return self._fatal(msg) # delete old folder if it exists and clean==True if ( DataManager.exists( parsed_uri=self._parsed_data_uris[self._source_context] ) and self._clean ): if not DataManager.delete( parsed_uri=self._parsed_data_uris[self._source_context] ): Log.a().warning( 'cannot delete existing data uri: %s', self._parsed_data_uris[self._source_context]['chopped_uri'] ) # create folder if not DataManager.mkdir( parsed_uri=self._parsed_data_uris[self._source_context], recursive=True ): msg = 'cannot create data uri: {}'.format( self._parsed_data_uris[self._source_context]['chopped_uri'] ) Log.an().error(msg) return self._fatal(msg) # create _log folder if not DataManager.mkdir( uri='{}/_log'.format( self._parsed_data_uris[self._source_context]['chopped_uri'] ), recursive=True ): msg = 'cannot create _log folder in data uri: {}/_log'.format( self._parsed_data_uris[self._source_context]['chopped_uri'] ) Log.an().error(msg) return self._fatal(msg) return True
def _run_map(self, map_item): """ Run a job for each map item and store the job ID. Args: self: class instance. map_item: map item object (item of self._map). Returns: On success: True. On failure: False. """ # load default app inputs overwrite with template inputs inputs = {} for input_key in self._app['inputs']: if input_key in map_item['template']: if map_item['template'][input_key]: # only include an input if the value is a non-empty string inputs[input_key] = urllib.parse.quote( str(map_item['template'][input_key]), safe='/:' ) else: if self._app['inputs'][input_key]['default']: # only include an input if the value is a non-empty string inputs[input_key] = urllib.parse.quote( str(self._app['inputs'][input_key]['default']), safe='/:' ) # load default app parameters, overwrite with template parameters parameters = {} for param_key in self._app['parameters']: if param_key in map_item['template']: if self._app['parameters'][param_key]['type'] in ['int', 'long']: parameters[param_key] = int(map_item['template'][param_key]) elif self._app['parameters'][param_key]['type'] == ['float', 'double']: parameters[param_key] = float(map_item['template'][param_key]) else: parameters[param_key] = str(map_item['template'][param_key]) else: if self._app['parameters'][param_key]['default'] not in [None, '']: parameters[param_key] \ = self._app['parameters'][param_key]['default'] # add execution method as parameter parameters['exec_method'] = self._step['execution']['method'] # add execution init commands if 'init' param given if 'init' in self._step['execution']['parameters']: parameters['exec_init'] = self._step['execution']['parameters']['init'] # construct agave app template name = 'gf-{}-{}-{}'.format( str(map_item['attempt']), slugify(self._step['name'], regex_pattern=r'[^-a-z0-9_]+'), slugify(map_item['template']['output'], regex_pattern=r'[^-a-z0-9_]+') ) name = name[:62]+'..' if len(name) > 64 else name archive_path = '{}/{}'.format( self._agave['parsed_archive_uri']['chopped_path'], name ) app_template = { 'name': name, 'appId': self._app['implementation']['agave']['agave_app_id'], 'archive': True, 'inputs': inputs, 'parameters': parameters, 'archiveSystem': self._agave['parsed_archive_uri']['authority'], 'archivePath': archive_path } # specify processors if 'slots' param given if 'slots' in self._step['execution']['parameters']: app_template['processorsPerNode'] = int( self._step['execution']['parameters']['slots'] ) # specify memory if 'mem' param given if 'mem' in self._step['execution']['parameters']: app_template['memoryPerNode'] = '{}'.format( self._step['execution']['parameters']['mem'] ) Log.some().debug( "[step.%s]: agave app template:\n%s", self._step['name'], pprint.pformat(app_template) ) # delete archive path if it exists if DataManager.exists( uri=self._agave['parsed_archive_uri']['chopped_uri']+'/'+name, agave=self._agave ): if not DataManager.delete( uri=self._agave['parsed_archive_uri']['chopped_uri']+'/'+name, agave=self._agave ): Log.a().warning( 'cannot delete archive uri: %s/%s', self._agave['parsed_archive_uri']['chopped_uri'], name ) # submit job job = self._agave['agave_wrapper'].jobs_submit(app_template) if not job: msg = 'agave jobs submit failed for "{}"'.format( app_template['name'] ) Log.an().error(msg) return self._fatal(msg) # log agave job id Log.some().debug( '[step.%s]: agave job id: %s -> %s', self._step['name'], map_item['template']['output'], job['id'] ) # record job info map_item['run'][map_item['attempt']]['agave_job_id'] = job['id'] map_item['run'][map_item['attempt']]['archive_uri'] = '{}/{}'\ .format( self._agave['parsed_archive_uri']['chopped_uri'], name ) map_item['run'][map_item['attempt']]['hpc_job_id'] = '' # set status of process map_item['status'] = 'PENDING' map_item['run'][map_item['attempt']]['status'] = 'PENDING' return True
def _run_map(self, map_item): """ Run a job for each map item and store the job ID. Args: self: class instance. map_item: map item object (item of self._map) Returns: On success: True. On failure: False. """ # load default app inputs overwrite with template inputs inputs = {} for input_key in self._app['inputs']: if input_key in map_item['template']: inputs[input_key] = urllib.parse.quote(str( map_item['template'][input_key] or ''), safe='/:') else: inputs[input_key] = urllib.parse.quote(str( self._app['inputs'][input_key]['default'] or ''), safe='/:') # load default app parameters, overwrite with template parameters parameters = {} for param_key in self._app['parameters']: if param_key in map_item['template']: parameters[param_key] = map_item['template'][param_key] else: parameters[param_key] \ = self._app['parameters'][param_key]['default'] # add execution method as parameter parameters['exec_method'] = self._step['execution']['method'] # construct agave app template name = 'gf-{}-{}-{}'.format(str(map_item['attempt']), slugify(self._step['name']), slugify(map_item['template']['output'])) name = name[:62] + '..' if len(name) > 64 else name archive_path = '{}/{}'.format( self._agave['parsed_archive_uri']['chopped_path'], name) app_template = { 'name': name, 'appId': self._app['definition']['agave']['agave_app_id'], 'archive': True, 'inputs': inputs, 'parameters': parameters, 'archiveSystem': self._agave['parsed_archive_uri']['authority'], 'archivePath': archive_path } Log.some().debug("agave app template:\n%s", pprint.pformat(app_template)) # delete archive path if it exists if DataManager.exists( uri=self._agave['parsed_archive_uri']['chopped_uri'] + '/' + name, agave=self._agave): if not DataManager.delete( uri=self._agave['parsed_archive_uri']['chopped_uri'] + '/' + name, agave=self._agave): Log.a().warning( 'cannot delete archive uri: %s/%s', self._agave['parsed_archive_uri']['chopped_uri'], name) # submit job job = self._agave['agave_wrapper'].jobs_submit(app_template) if not job: msg = 'agave jobs submit failed for "{}"'.format( app_template['name']) Log.an().error(msg) return self._fatal(msg) # log agave job id Log.some().debug('agave job id: %s -> %s', map_item['template']['output'], job['id']) # record job info map_item['run'][map_item['attempt']]['agave_job_id'] = job['id'] map_item['run'][map_item['attempt']]['archive_uri'] = '{}/{}'\ .format( self._agave['parsed_archive_uri']['chopped_uri'], name ) map_item['run'][map_item['attempt']]['hpc_job_id'] = '' # set status of process map_item['status'] = 'PENDING' map_item['run'][map_item['attempt']]['status'] = 'PENDING' return True
def register_agave_app(self, agave, agave_config, agave_params, agave_publish): """ Register app in Agave. Args: self: class instance Returns: On success: True. On failure: False. """ Log.some().info('registering agave app %s', str(self._path)) Log.some().info('app version: %s', self._config['version']) # compile agave app template if not TemplateCompiler.compile_template( self._path, 'agave-app-def.json.j2', self._path / 'agave-app-def.json', version=self._config['version'], agave=agave_params['agave'] ): Log.a().warning( 'cannot compile agave app "%s" definition from template', self._app['name'] ) return False # create main apps URI parsed_agave_apps_uri = URIParser.parse( 'agave://{}/{}'.format( agave_params['agave']['deploymentSystem'], agave_params['agave']['appsDir'] ) ) Log.some().info( 'creating main apps uri: %s', parsed_agave_apps_uri['chopped_uri'] ) if not DataManager.mkdir( parsed_uri=parsed_agave_apps_uri, recursive=True, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning('cannot create main agave apps uri') return False # delete app uri if it exists parsed_app_uri = URIParser.parse( 'agave://{}/{}/{}'.format( agave_params['agave']['deploymentSystem'], agave_params['agave']['appsDir'], self._app['folder'] ) ) Log.some().info( 'deleting app uri if it exists: %s', parsed_app_uri['chopped_uri'] ) if not DataManager.delete( parsed_uri=parsed_app_uri, agave={ 'agave': agave, 'agave_config': agave_config } ): # log warning, but ignore.. deleting non-existant uri returns False Log.a().warning( 'cannot delete app uri: %s', parsed_app_uri['chopped_uri'] ) # upload app assets parsed_assets_uri = URIParser.parse(str(self._path / 'assets')) Log.some().info( 'copying app assets from %s to %s', parsed_assets_uri['chopped_uri'], parsed_app_uri['chopped_uri'] ) if not DataManager.copy( parsed_src_uri=parsed_assets_uri, parsed_dest_uri=parsed_app_uri, local={}, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning( 'cannot copy app assets from %s to %s', parsed_assets_uri['chopped_uri'], parsed_app_uri['chopped_uri'] ) return False # upload test script parsed_test_uri = URIParser.parse( '{}/{}'.format( parsed_app_uri['chopped_uri'], 'test' ) ) Log.some().info( 'creating test uri: %s', parsed_test_uri['chopped_uri'] ) if not DataManager.mkdir( parsed_uri=parsed_test_uri, recursive=True, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning( 'cannot create test uri: %s', parsed_test_uri['chopped_uri'] ) return False parsed_local_test_script = URIParser.parse( str(self._path / 'test' / 'test.sh') ) parsed_agave_test_script = URIParser.parse( '{}/{}'.format(parsed_test_uri['chopped_uri'], 'test.sh') ) Log.some().info( 'copying test script from %s to %s', parsed_local_test_script['chopped_uri'], parsed_agave_test_script['chopped_uri'] ) if not DataManager.copy( parsed_src_uri=parsed_local_test_script, parsed_dest_uri=parsed_agave_test_script, local={}, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning( 'cannot copy test script from %s to %s', parsed_local_test_script['chopped_uri'], parsed_agave_test_script['chopped_uri'] ) return False # update existing app, or register new app Log.some().info('registering agave app') app_definition = self._yaml_to_dict( str(self._path / 'agave-app-def.json') ) if not app_definition: Log.a().warning( 'cannot load agave app definition: %s', str(self._path / 'agave-app-def.json') ) return False agwrap = AgaveAppsAddUpdate( agave, agave_config ) app_add_result = agwrap.call(app_definition) if not app_add_result: Log.a().warning( 'cannot register agave app:\n%s', pprint.pformat(app_definition) ) return False register_result = {} # publish app if agave_publish: Log.some().info('publishing agave app') agwrap = AgaveAppsPublish( agave, agave_config ) app_publish_result = agwrap.call(app_add_result['id']) if not app_publish_result: Log.a().warning( 'cannot publish agave app: %s', app_add_result['id'] ) return False # return published id and revision register_result = { 'id': app_publish_result['id'], 'version': self._config['version'], 'revision': 'u{}'.format(app_publish_result['revision']) } else: # return un-published id and blank revision register_result = { 'id': app_add_result['id'], 'version': self._config['version'], 'revision': '' } return register_result