def upload_agave_test_data(self): """ Upload Agave test data from workflow package. Args: self: class instance. Returns: None """ if (not self._agave or not self._agave_params or not self._agave_params.get('agave')): Log.a().warning( 'must provide agave parameters to upload test data') return False # create main test data URI parsed_base_test_uri = URIParser.parse('agave://{}/{}'.format( self._agave_params['agave']['deploymentSystem'], self._agave_params['agave']['testDataDir'])) Log.some().info('creating base test data uri: %s', parsed_base_test_uri['chopped_uri']) if not DataManager.mkdir(parsed_uri=parsed_base_test_uri, recursive=True, agave={ 'agave': self._agave, 'agave_config': self._config['agave'] }): Log.a().warning('cannot create base test data uri: %s', parsed_base_test_uri['chopped_uri']) return False # upload test data parsed_local_test_uri = URIParser.parse(str(Path(self._path) / 'data')) parsed_agave_test_uri = URIParser.parse('{}/{}'.format( parsed_base_test_uri['chopped_uri'], Path(self._path).name)) Log.some().info('copying test data from %s to %s', parsed_local_test_uri['chopped_uri'], parsed_agave_test_uri['chopped_uri']) if not DataManager.copy(parsed_src_uri=parsed_local_test_uri, parsed_dest_uri=parsed_agave_test_uri, local={}, agave={ 'agave': self._agave, 'agave_config': self._config['agave'] }): Log.a().warning('cannot copy test data from %s to %s', parsed_local_test_uri['chopped_uri'], parsed_agave_test_uri['chopped_uri']) return False return True
def stage(self, **kwargs): """ Copy data to all contexts except 'final' from source URI. Source URI can be multiple locations, but only copy to the first element of dest URIs. Set _staged indicator to True on success. Args: self: class instance. **kwargs: additional arguments required by DataManager.copy(). Returns: True or False. """ for context in self._parsed_data_uris: if context != self._source_context: if self._clean: # remove target URI first pass for i, parsed_source_uri in enumerate( self._parsed_data_uris[self._source_context]): Log.some().debug( 'staging data: %s->%s to %s->%s', self._source_context, parsed_source_uri['chopped_uri'], context, self._parsed_data_uris[context][i]['chopped_uri']) if context != 'final': if not DataManager.copy( parsed_src_uri=parsed_source_uri, parsed_dest_uri=self._parsed_data_uris[context] [i], **kwargs): msg = 'cannot stage data by copying from {} to {}'.format( parsed_source_uri['chopped_uri'], self._parsed_data_uris[context][i] ['chopped_uri']) Log.an().error(msg) return self._fatal(msg) self._staged = True return True
def stage(self, **kwargs): """ Copy data to all contexts except 'final' from source URI. Set _staged indicator to True on success. Args: self: class instance. **kwargs: additional arguments required by DataManager.copy(). Returns: True or False. """ for context in self._parsed_data_uris: if context != self._source_context: if self._clean: # remove target URI first pass Log.some().debug('staging data: {}->{} to {}->{}'.format( self._source_context, self._parsed_data_uris[ self._source_context]['chopped_uri'], context, self._parsed_data_uris[context]['chopped_uri'])) if context != 'final': if not DataManager.copy( parsed_src_uri=self._parsed_data_uris\ [self._source_context], parsed_dest_uri=self._parsed_data_uris[context], **kwargs ): msg = 'cannot stage data by copying from {} to {}'.format( self._parsed_data_uris[self._source_context]\ ['chopped_uri'], self._parsed_data_uris[context]['chopped_uri'] ) Log.an().error(msg) return self._fatal(msg) self._staged = True return True
def stage_final(self, **kwargs): """ Move data to final context from source URI. Set _staged_final indicator to True on success. Args: self: class instance. **kwargs: additional arguments required by DataManager.move(). Returns: True or False. """ for context in self._parsed_data_uris: if context != self._source_context: if self._clean: # remove target URI first pass Log.some().debug('staging final data: {}->{} to {}->{}'.format( self._source_context, self._parsed_data_uris[ self._source_context]['chopped_uri'], context, self._parsed_data_uris[context]['chopped_uri'])) if context == 'final': if (self._parsed_data_uris[self._source_context]['scheme'] == 'local' and self._parsed_data_uris[context]['scheme'] == 'local'): # move final data instead of copy, only for local-->local schemes if not DataManager.move( parsed_src_uri=self._parsed_data_uris\ [self._source_context], parsed_dest_uri=self._parsed_data_uris[context], **kwargs ): msg = 'cannot stage final data by copying from {} to {}'.format( self._parsed_data_uris[self._source_context]\ ['chopped_uri'], self._parsed_data_uris[context]['chopped_uri'] ) Log.an().error(msg) return self._fatal(msg) else: if not DataManager.copy( parsed_src_uri=self._parsed_data_uris\ [self._source_context], parsed_dest_uri=self._parsed_data_uris[context], **kwargs ): msg = 'cannot stage final data by copying from {} to {}'.format( self._parsed_data_uris[self._source_context]\ ['chopped_uri'], self._parsed_data_uris[context]['chopped_uri'] ) Log.an().error(msg) return self._fatal(msg) self._staged_final = True return True
def register_agave_app(self, agave, agave_config, agave_params, agave_publish): """ Register app in Agave. Args: self: class instance Returns: On success: True. On failure: False. """ Log.some().info('registering agave app %s', str(self._path)) Log.some().info('app version: %s', self._config['version']) # compile agave app template if not TemplateCompiler.compile_template( self._path, 'agave-app-def.json.j2', self._path / 'agave-app-def.json', version=self._config['version'], agave=agave_params['agave'] ): Log.a().warning( 'cannot compile agave app "%s" definition from template', self._app['name'] ) return False # create main apps URI parsed_agave_apps_uri = URIParser.parse( 'agave://{}/{}'.format( agave_params['agave']['deploymentSystem'], agave_params['agave']['appsDir'] ) ) Log.some().info( 'creating main apps uri: %s', parsed_agave_apps_uri['chopped_uri'] ) if not DataManager.mkdir( parsed_uri=parsed_agave_apps_uri, recursive=True, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning('cannot create main agave apps uri') return False # delete app uri if it exists parsed_app_uri = URIParser.parse( 'agave://{}/{}/{}'.format( agave_params['agave']['deploymentSystem'], agave_params['agave']['appsDir'], self._app['folder'] ) ) Log.some().info( 'deleting app uri if it exists: %s', parsed_app_uri['chopped_uri'] ) if not DataManager.delete( parsed_uri=parsed_app_uri, agave={ 'agave': agave, 'agave_config': agave_config } ): # log warning, but ignore.. deleting non-existant uri returns False Log.a().warning( 'cannot delete app uri: %s', parsed_app_uri['chopped_uri'] ) # upload app assets parsed_assets_uri = URIParser.parse(str(self._path / 'assets')) Log.some().info( 'copying app assets from %s to %s', parsed_assets_uri['chopped_uri'], parsed_app_uri['chopped_uri'] ) if not DataManager.copy( parsed_src_uri=parsed_assets_uri, parsed_dest_uri=parsed_app_uri, local={}, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning( 'cannot copy app assets from %s to %s', parsed_assets_uri['chopped_uri'], parsed_app_uri['chopped_uri'] ) return False # upload test script parsed_test_uri = URIParser.parse( '{}/{}'.format( parsed_app_uri['chopped_uri'], 'test' ) ) Log.some().info( 'creating test uri: %s', parsed_test_uri['chopped_uri'] ) if not DataManager.mkdir( parsed_uri=parsed_test_uri, recursive=True, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning( 'cannot create test uri: %s', parsed_test_uri['chopped_uri'] ) return False parsed_local_test_script = URIParser.parse( str(self._path / 'test' / 'test.sh') ) parsed_agave_test_script = URIParser.parse( '{}/{}'.format(parsed_test_uri['chopped_uri'], 'test.sh') ) Log.some().info( 'copying test script from %s to %s', parsed_local_test_script['chopped_uri'], parsed_agave_test_script['chopped_uri'] ) if not DataManager.copy( parsed_src_uri=parsed_local_test_script, parsed_dest_uri=parsed_agave_test_script, local={}, agave={ 'agave': agave, 'agave_config': agave_config } ): Log.a().warning( 'cannot copy test script from %s to %s', parsed_local_test_script['chopped_uri'], parsed_agave_test_script['chopped_uri'] ) return False # update existing app, or register new app Log.some().info('registering agave app') app_definition = self._yaml_to_dict( str(self._path / 'agave-app-def.json') ) if not app_definition: Log.a().warning( 'cannot load agave app definition: %s', str(self._path / 'agave-app-def.json') ) return False agwrap = AgaveAppsAddUpdate( agave, agave_config ) app_add_result = agwrap.call(app_definition) if not app_add_result: Log.a().warning( 'cannot register agave app:\n%s', pprint.pformat(app_definition) ) return False register_result = {} # publish app if agave_publish: Log.some().info('publishing agave app') agwrap = AgaveAppsPublish( agave, agave_config ) app_publish_result = agwrap.call(app_add_result['id']) if not app_publish_result: Log.a().warning( 'cannot publish agave app: %s', app_add_result['id'] ) return False # return published id and revision register_result = { 'id': app_publish_result['id'], 'version': self._config['version'], 'revision': 'u{}'.format(app_publish_result['revision']) } else: # return un-published id and blank revision register_result = { 'id': app_add_result['id'], 'version': self._config['version'], 'revision': '' } return register_result