def upload_agave_test_data(self):
"""
Upload Agave test data from workflow package.
Args:
self: class instance.
Returns:
None
"""
if (not self._agave or not self._agave_params
or not self._agave_params.get('agave')):
Log.a().warning(
'must provide agave parameters to upload test data')
return False
# create main test data URI
parsed_base_test_uri = URIParser.parse('agave://{}/{}'.format(
self._agave_params['agave']['deploymentSystem'],
self._agave_params['agave']['testDataDir']))
Log.some().info('creating base test data uri: %s',
parsed_base_test_uri['chopped_uri'])
if not DataManager.mkdir(parsed_uri=parsed_base_test_uri,
recursive=True,
agave={
'agave': self._agave,
'agave_config': self._config['agave']
}):
Log.a().warning('cannot create base test data uri: %s',
parsed_base_test_uri['chopped_uri'])
return False
# upload test data
parsed_local_test_uri = URIParser.parse(str(Path(self._path) / 'data'))
parsed_agave_test_uri = URIParser.parse('{}/{}'.format(
parsed_base_test_uri['chopped_uri'],
Path(self._path).name))
Log.some().info('copying test data from %s to %s',
parsed_local_test_uri['chopped_uri'],
parsed_agave_test_uri['chopped_uri'])
if not DataManager.copy(parsed_src_uri=parsed_local_test_uri,
parsed_dest_uri=parsed_agave_test_uri,
local={},
agave={
'agave': self._agave,
'agave_config': self._config['agave']
}):
Log.a().warning('cannot copy test data from %s to %s',
parsed_local_test_uri['chopped_uri'],
parsed_agave_test_uri['chopped_uri'])
return False
return True
def stage(self, **kwargs):
"""
Copy data to all contexts except 'final' from source URI. Source URI can be multiple
locations, but only copy to the first element of dest URIs.
Set _staged indicator to True on success.
Args:
self: class instance.
**kwargs: additional arguments required by DataManager.copy().
Returns:
True or False.
"""
for context in self._parsed_data_uris:
if context != self._source_context:
if self._clean:
# remove target URI first
pass
for i, parsed_source_uri in enumerate(
self._parsed_data_uris[self._source_context]):
Log.some().debug(
'staging data: %s->%s to %s->%s', self._source_context,
parsed_source_uri['chopped_uri'], context,
self._parsed_data_uris[context][i]['chopped_uri'])
if context != 'final':
if not DataManager.copy(
parsed_src_uri=parsed_source_uri,
parsed_dest_uri=self._parsed_data_uris[context]
[i],
**kwargs):
msg = 'cannot stage data by copying from {} to {}'.format(
parsed_source_uri['chopped_uri'],
self._parsed_data_uris[context][i]
['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
self._staged = True
return True
def stage(self, **kwargs):
"""
Copy data to all contexts except 'final' from source URI.
Set _staged indicator to True on success.
Args:
self: class instance.
**kwargs: additional arguments required by DataManager.copy().
Returns:
True or False.
"""
for context in self._parsed_data_uris:
if context != self._source_context:
if self._clean:
# remove target URI first
pass
Log.some().debug('staging data: {}->{} to {}->{}'.format(
self._source_context, self._parsed_data_uris[
self._source_context]['chopped_uri'], context,
self._parsed_data_uris[context]['chopped_uri']))
if context != 'final':
if not DataManager.copy(
parsed_src_uri=self._parsed_data_uris\
[self._source_context],
parsed_dest_uri=self._parsed_data_uris[context],
**kwargs
):
msg = 'cannot stage data by copying from {} to {}'.format(
self._parsed_data_uris[self._source_context]\
['chopped_uri'],
self._parsed_data_uris[context]['chopped_uri']
)
Log.an().error(msg)
return self._fatal(msg)
self._staged = True
return True
def stage_final(self, **kwargs):
"""
Move data to final context from source URI.
Set _staged_final indicator to True on success.
Args:
self: class instance.
**kwargs: additional arguments required by DataManager.move().
Returns:
True or False.
"""
for context in self._parsed_data_uris:
if context != self._source_context:
if self._clean:
# remove target URI first
pass
Log.some().debug('staging final data: {}->{} to {}->{}'.format(
self._source_context, self._parsed_data_uris[
self._source_context]['chopped_uri'], context,
self._parsed_data_uris[context]['chopped_uri']))
if context == 'final':
if (self._parsed_data_uris[self._source_context]['scheme']
== 'local'
and self._parsed_data_uris[context]['scheme']
== 'local'):
# move final data instead of copy, only for local-->local schemes
if not DataManager.move(
parsed_src_uri=self._parsed_data_uris\
[self._source_context],
parsed_dest_uri=self._parsed_data_uris[context],
**kwargs
):
msg = 'cannot stage final data by copying from {} to {}'.format(
self._parsed_data_uris[self._source_context]\
['chopped_uri'],
self._parsed_data_uris[context]['chopped_uri']
)
Log.an().error(msg)
return self._fatal(msg)
else:
if not DataManager.copy(
parsed_src_uri=self._parsed_data_uris\
[self._source_context],
parsed_dest_uri=self._parsed_data_uris[context],
**kwargs
):
msg = 'cannot stage final data by copying from {} to {}'.format(
self._parsed_data_uris[self._source_context]\
['chopped_uri'],
self._parsed_data_uris[context]['chopped_uri']
)
Log.an().error(msg)
return self._fatal(msg)
self._staged_final = True
return True
def register_agave_app(self, agave, agave_config, agave_params, agave_publish):
"""
Register app in Agave.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
Log.some().info('registering agave app %s', str(self._path))
Log.some().info('app version: %s', self._config['version'])
# compile agave app template
if not TemplateCompiler.compile_template(
self._path,
'agave-app-def.json.j2',
self._path / 'agave-app-def.json',
version=self._config['version'],
agave=agave_params['agave']
):
Log.a().warning(
'cannot compile agave app "%s" definition from template',
self._app['name']
)
return False
# create main apps URI
parsed_agave_apps_uri = URIParser.parse(
'agave://{}/{}'.format(
agave_params['agave']['deploymentSystem'],
agave_params['agave']['appsDir']
)
)
Log.some().info(
'creating main apps uri: %s',
parsed_agave_apps_uri['chopped_uri']
)
if not DataManager.mkdir(
parsed_uri=parsed_agave_apps_uri,
recursive=True,
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning('cannot create main agave apps uri')
return False
# delete app uri if it exists
parsed_app_uri = URIParser.parse(
'agave://{}/{}/{}'.format(
agave_params['agave']['deploymentSystem'],
agave_params['agave']['appsDir'],
self._app['folder']
)
)
Log.some().info(
'deleting app uri if it exists: %s',
parsed_app_uri['chopped_uri']
)
if not DataManager.delete(
parsed_uri=parsed_app_uri,
agave={
'agave': agave,
'agave_config': agave_config
}
):
# log warning, but ignore.. deleting non-existant uri returns False
Log.a().warning(
'cannot delete app uri: %s', parsed_app_uri['chopped_uri']
)
# upload app assets
parsed_assets_uri = URIParser.parse(str(self._path / 'assets'))
Log.some().info(
'copying app assets from %s to %s',
parsed_assets_uri['chopped_uri'],
parsed_app_uri['chopped_uri']
)
if not DataManager.copy(
parsed_src_uri=parsed_assets_uri,
parsed_dest_uri=parsed_app_uri,
local={},
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning(
'cannot copy app assets from %s to %s',
parsed_assets_uri['chopped_uri'],
parsed_app_uri['chopped_uri']
)
return False
# upload test script
parsed_test_uri = URIParser.parse(
'{}/{}'.format(
parsed_app_uri['chopped_uri'],
'test'
)
)
Log.some().info(
'creating test uri: %s', parsed_test_uri['chopped_uri']
)
if not DataManager.mkdir(
parsed_uri=parsed_test_uri,
recursive=True,
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning(
'cannot create test uri: %s', parsed_test_uri['chopped_uri']
)
return False
parsed_local_test_script = URIParser.parse(
str(self._path / 'test' / 'test.sh')
)
parsed_agave_test_script = URIParser.parse(
'{}/{}'.format(parsed_test_uri['chopped_uri'], 'test.sh')
)
Log.some().info(
'copying test script from %s to %s',
parsed_local_test_script['chopped_uri'],
parsed_agave_test_script['chopped_uri']
)
if not DataManager.copy(
parsed_src_uri=parsed_local_test_script,
parsed_dest_uri=parsed_agave_test_script,
local={},
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning(
'cannot copy test script from %s to %s',
parsed_local_test_script['chopped_uri'],
parsed_agave_test_script['chopped_uri']
)
return False
# update existing app, or register new app
Log.some().info('registering agave app')
app_definition = self._yaml_to_dict(
str(self._path / 'agave-app-def.json')
)
if not app_definition:
Log.a().warning(
'cannot load agave app definition: %s',
str(self._path / 'agave-app-def.json')
)
return False
agwrap = AgaveAppsAddUpdate(
agave, agave_config
)
app_add_result = agwrap.call(app_definition)
if not app_add_result:
Log.a().warning(
'cannot register agave app:\n%s', pprint.pformat(app_definition)
)
return False
register_result = {}
# publish app
if agave_publish:
Log.some().info('publishing agave app')
agwrap = AgaveAppsPublish(
agave, agave_config
)
app_publish_result = agwrap.call(app_add_result['id'])
if not app_publish_result:
Log.a().warning(
'cannot publish agave app: %s', app_add_result['id']
)
return False
# return published id and revision
register_result = {
'id': app_publish_result['id'],
'version': self._config['version'],
'revision': 'u{}'.format(app_publish_result['revision'])
}
else:
# return un-published id and blank revision
register_result = {
'id': app_add_result['id'],
'version': self._config['version'],
'revision': ''
}
return register_result