def make_test(self):
"""
Generate the GeneFlow app test script.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
# make test folder, if it doesn't already exist
test_path = Path(self._path / 'test')
test_path.mkdir(exist_ok=True)
Log.some().info('compiling %s', str(test_path / 'test.sh'))
# compile jinja2 template
if not TemplateCompiler.compile_template(
None,
'test.sh.j2',
str(test_path / 'test.sh'),
**self._config
):
Log.an().error('cannot compile GeneFlow app test script')
return False
return True
def make_wrapper(self):
"""
Generate the GeneFlow app wrapper script.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
# make assets folder, if it doesn't already exist
asset_path = Path(self._path / 'assets')
asset_path.mkdir(exist_ok=True)
Log.some().info(
'compiling %s',
str(asset_path / '{}.sh'.format(self._config['name']))
)
# compile jinja2 template
if not TemplateCompiler.compile_template(
None,
'wrapper-script.sh.j2',
str(asset_path / '{}.sh'.format(self._config['name'])),
**self._config
):
Log.an().error('cannot compile GeneFlow app wrapper script')
return False
return True
def make_wrapper(self):
"""
Generate the GeneFlow app wrapper script.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
# make assets folder, if it doesn't already exist
asset_path = Path(self._path / 'assets')
asset_path.mkdir(exist_ok=True)
script_path = str(asset_path / '{}.sh'.format(
slugify(self._app['name'], regex_pattern=r'[^-a-z0-9_]+')))
Log.some().info('compiling %s', script_path)
# compile jinja2 template
if not TemplateCompiler.compile_template(None, 'wrapper-script.sh.j2',
script_path, **self._app):
Log.an().error('cannot compile GeneFlow app wrapper script')
return False
# make script executable by owner
os.chmod(script_path, stat.S_IRWXU)
return True
def make_agave(self):
"""
Generate the GeneFlow Agave app definition.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
Log.some().info('compiling %s',
str(self._path / 'agave-app-def.json.j2'))
if not TemplateCompiler.compile_template(
None,
'agave-app-def.json.j2.j2',
str(self._path / 'agave-app-def.json.j2'),
slugify_name=slugify(self._app['name'],
regex_pattern=r'[^-a-z0-9_]+'),
**self._app):
Log.an().error(
'cannot compile GeneFlow Agave app definition template')
return False
return True
def make_def(self):
"""
Generate the GeneFlow app definition.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
Log.some().info('compiling %s', str(self._path / 'app.yaml.j2'))
if not TemplateCompiler.compile_template(
None, 'app.yaml.j2.j2', str(self._path / 'app.yaml.j2'), **
self._config):
Log.an().error('cannot compile GeneFlow app definition template')
return False
return True
def install_apps(self):
"""
Install apps for the workflow package.
Args:
self: class instance.
Returns:
None
"""
apps_path = Path(self._path) / 'workflow' / 'apps'
if self._clean:
# remove apps folder
if apps_path.is_dir():
shutil.rmtree(str(apps_path))
# create apps folder if not already there
apps_path.mkdir(exist_ok=True)
for app in self._apps_repo['apps']:
if self._app_name == app['name'] or not self._app_name:
Log.some().info('app:\n%s', pprint.pformat(app))
repo_path = apps_path / app['folder']
# create AppInstaller instance
app_installer = AppInstaller(str(repo_path), app,
self._app_asset,
self._copy_prefix)
# clone app into install location
if not app_installer.clone_git_repo():
Log.an().error('cannot clone app to %s', str(repo_path))
return False
if not app_installer.load_config():
Log.an().error('cannot load app config.yaml')
return False
if self._make_apps:
if not app_installer.make():
Log.an().error('cannot compile app templates')
return False
if not app_installer.install_assets():
Log.an().error('cannot install app assets')
return False
# register in Agave
if (self._agave_wrapper and self._agave_params
and self._agave_params.get('agave')):
register_result = app_installer.register_agave_app(
self._agave_wrapper, self._agave_params,
self._agave_publish)
if not register_result:
Log.an().error('cannot register app "%s" in agave',
app['name'])
return False
Log.some().info('registered agave app:\n%s',
pprint.pformat(register_result))
# compile jinja template for published app definition
if not TemplateCompiler.compile_template(
repo_path,
'app.yaml.j2',
repo_path / 'app.yaml',
agave=self._agave_params['agave'],
version=register_result['version'],
revision=register_result['revision']):
Log.an().error(
'cannot compile app "%s" definition from template',
app['name'])
return False
else:
# compile jinja template for app definition
if not TemplateCompiler.compile_template(
repo_path, 'app.yaml.j2', repo_path / 'app.yaml'):
Log.an().error(
'cannot compile app "%s" definition from template',
app['name'])
return False
return True
def register_agave_app(self, agave, agave_config, agave_params, agave_publish):
"""
Register app in Agave.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
Log.some().info('registering agave app %s', str(self._path))
Log.some().info('app version: %s', self._config['version'])
# compile agave app template
if not TemplateCompiler.compile_template(
self._path,
'agave-app-def.json.j2',
self._path / 'agave-app-def.json',
version=self._config['version'],
agave=agave_params['agave']
):
Log.a().warning(
'cannot compile agave app "%s" definition from template',
self._app['name']
)
return False
# create main apps URI
parsed_agave_apps_uri = URIParser.parse(
'agave://{}/{}'.format(
agave_params['agave']['deploymentSystem'],
agave_params['agave']['appsDir']
)
)
Log.some().info(
'creating main apps uri: %s',
parsed_agave_apps_uri['chopped_uri']
)
if not DataManager.mkdir(
parsed_uri=parsed_agave_apps_uri,
recursive=True,
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning('cannot create main agave apps uri')
return False
# delete app uri if it exists
parsed_app_uri = URIParser.parse(
'agave://{}/{}/{}'.format(
agave_params['agave']['deploymentSystem'],
agave_params['agave']['appsDir'],
self._app['folder']
)
)
Log.some().info(
'deleting app uri if it exists: %s',
parsed_app_uri['chopped_uri']
)
if not DataManager.delete(
parsed_uri=parsed_app_uri,
agave={
'agave': agave,
'agave_config': agave_config
}
):
# log warning, but ignore.. deleting non-existant uri returns False
Log.a().warning(
'cannot delete app uri: %s', parsed_app_uri['chopped_uri']
)
# upload app assets
parsed_assets_uri = URIParser.parse(str(self._path / 'assets'))
Log.some().info(
'copying app assets from %s to %s',
parsed_assets_uri['chopped_uri'],
parsed_app_uri['chopped_uri']
)
if not DataManager.copy(
parsed_src_uri=parsed_assets_uri,
parsed_dest_uri=parsed_app_uri,
local={},
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning(
'cannot copy app assets from %s to %s',
parsed_assets_uri['chopped_uri'],
parsed_app_uri['chopped_uri']
)
return False
# upload test script
parsed_test_uri = URIParser.parse(
'{}/{}'.format(
parsed_app_uri['chopped_uri'],
'test'
)
)
Log.some().info(
'creating test uri: %s', parsed_test_uri['chopped_uri']
)
if not DataManager.mkdir(
parsed_uri=parsed_test_uri,
recursive=True,
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning(
'cannot create test uri: %s', parsed_test_uri['chopped_uri']
)
return False
parsed_local_test_script = URIParser.parse(
str(self._path / 'test' / 'test.sh')
)
parsed_agave_test_script = URIParser.parse(
'{}/{}'.format(parsed_test_uri['chopped_uri'], 'test.sh')
)
Log.some().info(
'copying test script from %s to %s',
parsed_local_test_script['chopped_uri'],
parsed_agave_test_script['chopped_uri']
)
if not DataManager.copy(
parsed_src_uri=parsed_local_test_script,
parsed_dest_uri=parsed_agave_test_script,
local={},
agave={
'agave': agave,
'agave_config': agave_config
}
):
Log.a().warning(
'cannot copy test script from %s to %s',
parsed_local_test_script['chopped_uri'],
parsed_agave_test_script['chopped_uri']
)
return False
# update existing app, or register new app
Log.some().info('registering agave app')
app_definition = self._yaml_to_dict(
str(self._path / 'agave-app-def.json')
)
if not app_definition:
Log.a().warning(
'cannot load agave app definition: %s',
str(self._path / 'agave-app-def.json')
)
return False
agwrap = AgaveAppsAddUpdate(
agave, agave_config
)
app_add_result = agwrap.call(app_definition)
if not app_add_result:
Log.a().warning(
'cannot register agave app:\n%s', pprint.pformat(app_definition)
)
return False
register_result = {}
# publish app
if agave_publish:
Log.some().info('publishing agave app')
agwrap = AgaveAppsPublish(
agave, agave_config
)
app_publish_result = agwrap.call(app_add_result['id'])
if not app_publish_result:
Log.a().warning(
'cannot publish agave app: %s', app_add_result['id']
)
return False
# return published id and revision
register_result = {
'id': app_publish_result['id'],
'version': self._config['version'],
'revision': 'u{}'.format(app_publish_result['revision'])
}
else:
# return un-published id and blank revision
register_result = {
'id': app_add_result['id'],
'version': self._config['version'],
'revision': ''
}
return register_result