def _init_data_uri(self):
"""
Create output data URI for the source context (local).
Args:
self: class instance.
Returns:
On success: True.
On failure: False.
"""
# make sure the source data URI has a compatible scheme (local)
if self._parsed_data_uris[self._source_context]['scheme'] != 'local':
msg = 'invalid data uri scheme for this step: {}'.format(
self._parsed_data_uris[self._source_context]['scheme'])
Log.an().error(msg)
return self._fatal(msg)
# delete old folder if it exists and clean==True
if (DataManager.exists(
parsed_uri=self._parsed_data_uris[self._source_context])
and self._clean):
if not DataManager.delete(
parsed_uri=self._parsed_data_uris[self._source_context]):
Log.a().warning(
'cannot delete existing data uri: %s',
self._parsed_data_uris[
self._source_context]['chopped_uri'])
# create folder
if not DataManager.mkdir(
parsed_uri=self._parsed_data_uris[self._source_context],
recursive=True):
msg = 'cannot create data uri: {}'.format(
self._parsed_data_uris[self._source_context]['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
# create _log folder
if not DataManager.mkdir(uri='{}/_log'.format(
self._parsed_data_uris[self._source_context]['chopped_uri']),
recursive=True):
msg = 'cannot create _log folder in data uri: {}/_log'.format(
self._parsed_data_uris[self._source_context]['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
return True
def files_import_from_agave(
self, system_id, file_path, file_name, url_to_ingest
):
"""
Wrap AgavePy import data file command.
Args:
self: class instance.
system_id: Identifier for Agave storage system.
file_path: Path where file is to be imported.
file_name: Name of the imported file.
url_to_ingest: Agave URL to be ingested.
Returns:
On success: True with no exceptions.
On failure: Throws exception.
"""
response = self._agave.files.importData(
systemId=system_id,
filePath=file_path,
fileName=file_name,
urlToIngest=urllib.parse.quote(str(url_to_ingest or ''), safe='/:')
)
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().debug(
'import %s: %s -> agave://%s/%s/%s',
str(status),
url_to_ingest,
system_id,
file_path,
file_name
)
if str(status) == 'FINISHED':
return True
# not finished, try again
raise Exception('agave import failed')
def resolve_workflow_path(workflow_identifier):
"""
Search GENEFLOW_PATH env var to find workflow definition.
Args:
workflow_identifier: workflow identifier
Returns:
On success: Full path of workflow yaml file (str).
On failure: False.
"""
# check if abs path or in current directory first (.)
abs_path = Path.absolute(Path(workflow_identifier))
if abs_path.is_file():
return str(abs_path)
if abs_path.is_dir(): # assume this is the name of workflow package dir
yaml_path = Path(abs_path / 'workflow' / 'workflow.yaml')
if yaml_path.is_file():
return str(yaml_path)
# search GENEFLOW_PATH
gf_path = os.environ.get('GENEFLOW_PATH')
if gf_path:
for path in gf_path.split(':'):
if path:
wf_path = Path(path) / workflow_identifier
if wf_path.is_dir():
yaml_path = Path(wf_path / 'workflow' / 'workflow.yaml')
if yaml_path.is_file():
return str(yaml_path)
Log.an().error(
'workflow "%s" not found, check GENEFLOW_PATH', workflow_identifier
)
return False
def _init_workflow_context_data(self):
"""
Initialize data components of workflow contexts.
Args:
None.
Returns:
On success: True.
On failure: False.
"""
for exec_context in self._exec_contexts:
if not self._workflow_context[exec_context].init_data():
msg = (
'cannot initialize data for workflow context: {}'\
.format(exec_context)
)
Log.an().error(msg)
return self._fatal(msg)
return True
def _mkdir_agave(uri, agave):
"""
Create agave directory specified by URI.
Args:
uri: parsed URI to create.
agave: dict that contains:
agave: AgavePy connection object.
agave_config: Agave connection info, needed by AgavePyWrapper.
Returns:
On success: True.
On failure: False.
"""
agwrap = AgaveFilesMkDir(agave['agave'], agave['agave_config'])
if not agwrap.call(uri['authority'], uri['folder'], uri['name']):
Log.an().error('cannot create folder at uri: %s', uri['chopped_uri'])
return False
return True
def _delete_agave(uri, agave):
"""
Delete agave file/folder specified by URI.
Args:
uri: parsed URI to delete.
agave: dict that contains:
agave: AgavePy connection object.
agave_config: Agave connection info, needed by AgavePyWrapper.
Returns:
On success: True.
On failure: False.
"""
agwrap = AgaveFilesDelete(agave['agave'], agave['agave_config'])
if not agwrap.call(uri['authority'], uri['chopped_path']):
Log.an().error('cannot delete uri: %s', uri['chopped_path'])
return False
return True
def _mkdir_local(uri, local=None):
"""
Create local directory specified by URI.
Args:
uri: parsed URI to create.
local: local context options.
Returns:
On success: True.
On failure: False.
"""
try:
os.makedirs(uri['chopped_path'])
except OSError as err:
Log.an().error('cannot create uri: %s [%s]', uri['chopped_uri'],
str(err))
return False
return True
def clone_git_repo(self):
"""
Clone app from git repo.
Args:
self: class instance
Returns:
On success: True
On failure: False
"""
# remove app folder if it exists
if self._path.is_dir():
shutil.rmtree(str(self._path))
# recreate app folder
self._path.mkdir()
# clone app's git repo into target location
try:
if self._app['tag']:
Repo.clone_from(
self._app['repo'], str(self._path), branch=self._app['tag'],
config='http.sslVerify=false'
)
else:
Repo.clone_from(
self._app['repo'], str(self._path),
config='http.sslVerify=false'
)
except GitError as err:
Log.an().error(
'cannot clone app git repo: %s [%s]',
self._app['repo'], str(err)
)
return False
return True
def write(self, config_path):
"""
Write current config to file.
Args:
config_path: file to write config.
Returns:
On success: True.
On failure: False.
"""
try:
with open(config_path, 'w') as out_file:
yaml.dump(self._config, out_file, default_flow_style=False)
except IOError as err:
Log.an().error('cannot write yaml file to %s [%s]', config_path,
str(err))
return False
return True
def _delete_local(uri, local=None):
"""
Delete local file/folder specified by URI.
Args:
uri: parsed URI to delete.
local: local context options.
Returns:
On success: True.
On failure: False.
"""
try:
shutil.rmtree(uri['chopped_path'])
except OSError as err:
Log.an().error('cannot delete uri: %s [%s]', uri['chopped_uri'],
str(err))
return False
return True
def _list_agave(uri, agave):
"""
List contents of agave URI.
Args:
uri: parsed URI to list.
agave: dict that contains:
agave_wrapper: Agave wrapper object.
Returns:
On success: a list of filenames (basenames only).
On failure: False.
"""
file_list = agave['agave_wrapper'].files_list(uri['authority'],
uri['chopped_path'])
if file_list is False:
Log.an().error('cannot get file list for uri: %s', uri['chopped_uri'])
return False
return [file['name'] for file in file_list]
def _move_local_local(src_uri, dest_uri, local=None):
"""
Move local data with system shell.
Args:
src_uri: Source URI parsed into dict with URIParser.
dest_uri: Destination URI parsed into dict with URIParser.
local: local context options.
Returns:
On success: True.
On failure: False.
"""
try:
shutil.move(src_uri['path'], dest_uri['path'])
except OSError as err:
Log.an().error('cannot move from %s to %s [%s]', src_uri['uri'],
dest_uri['uri'], str(err))
return False
return True
def check_running_jobs(self):
"""
Check the status/progress of all map-reduce items and update _map status.
Args:
self: class instance.
Returns:
True.
"""
# check if jobs are running, finished, or failed
for map_item in self._map:
if map_item['status'] not in ['FINISHED', 'FAILED', 'PENDING']:
try:
# can only get job status if it has not already been disposed with "wait"
status = self._slurm['drmaa_session'].jobStatus(
map_item['run'][map_item['attempt']]['hpc_job_id'])
map_item['status'] = self._job_status_map[status]
except drmaa.DrmCommunicationException as err:
msg = 'cannot get job status for step "{}" [{}]'\
.format(self._step['name'], str(err))
Log.a().warning(msg)
map_item['status'] = 'UNKNOWN'
if map_item['status'] in ['FINISHED', 'FAILED']:
# check exit status
job_info = self._slurm['drmaa_session'].wait(
map_item['run'][map_item['attempt']]['hpc_job_id'],
self._slurm['drmaa_session'].TIMEOUT_NO_WAIT)
Log.a().debug('[step.%s]: exit status: %s -> %s',
self._step['name'],
map_item['template']['output'],
job_info.exitStatus)
if job_info.exitStatus > 0:
# job actually failed
map_item['status'] = 'FAILED'
# decrease num running procs
if self._num_running > 0:
self._num_running -= 1
map_item['run'][map_item['attempt']]['status'] = map_item['status']
if map_item['status'] == 'FAILED' and map_item['attempt'] < 5:
if self._throttle_limit == 0 or self._num_running < self._throttle_limit:
# retry job if not at retry or throttle limit
if not self.retry_failed(map_item):
Log.a().warning(
'[step.%s]: cannot retry failed slurm job (%s)',
self._step['name'], map_item['template']['output'])
else:
self._num_running += 1
self._update_status_db(self._status, '')
return True
def load_config(self):
"""
Load app config.yaml file.
Args:
self: class instance
Returns:
On success: True
On failure: False
"""
# read yaml file
self._config = self._yaml_to_dict(
str(Path(self._path / 'config.yaml'))
)
# empty dict?
if not self._config:
Log.an().error(
'cannot load/parse config.yaml file in app: %s', self._path
)
return False
validator = cerberus.Validator(allow_unknown=False)
valid_def = validator.validated(
self._config,
CONFIG_SCHEMA[GF_VERSION]
)
if not valid_def:
Log.an().error(
'app config validation error:\n%s',
pprint.pformat(validator.errors)
)
return False
return True
def _load_context_classes(self):
"""
Import modules and load references to step classes.
Dynamically load modules for step classes, and keep a reference to the
step class in the _context_classes dict.
Args:
None.
Returns:
On failure: Raises WorkflowDAGException.
"""
for context in self._parsed_job_work_uri:
mod_name = '{}_step'.format(context)
cls_name = '{}Step'.format(context.capitalize())
try:
step_mod = __import__('geneflow.extend.{}'.format(mod_name),
fromlist=[mod_name])
except ImportError as err:
msg = 'cannot import context-specific step module: {} [{}]'\
.format(mod_name, str(err))
Log.an().error(msg)
raise WorkflowDAGException(msg)
try:
step_class = getattr(step_mod, cls_name)
except AttributeError as err:
msg = 'cannot import context-specific step class: {} [{}]'\
.format(cls_name, str(err))
Log.an().error(msg)
raise WorkflowDAGException(msg)
# reference to step class
self._context_classes[context] = step_class
def _validate_depend_uris(self):
"""
Validate the depend URI list against depend list of the step definition.
Also validate that the scheme of each depend URI matches
that of this step's output (i.e., the data URI of the source context.
Args:
self: class instance.
Returns:
On success: True.
On failure: False.
"""
for depend in self._step['depend']:
if depend in self._depend_uris:
# check if depend URI scheme is the same as that of this
# step's output
if (
self._depend_uris[depend]['scheme']
!= self._parsed_data_uris[self._source_context]\
['scheme']
):
msg = 'incompatible scheme for depend uri {} of step {}'\
.format(
self._depend_uris[depend]['chopped_uri'],
depend
)
Log.an().error(msg)
return self._fatal(msg)
else:
msg = 'uri missing for dependent step {}'.format(depend)
Log.an().error(msg)
return self._fatal(msg)
return True
def make_agave(self):
"""
Generate the GeneFlow Agave app definition.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
Log.some().info('compiling %s',
str(self._path / 'agave-app-def.json.j2'))
if not TemplateCompiler.compile_template(
None, 'agave-app-def.json.j2.j2',
str(self._path / 'agave-app-def.json.j2'), **self._config):
Log.an().error(
'cannot compile GeneFlow Agave app definition template')
return False
return True
def _clone_workflow(self):
if not self._git:
Log.an().error('must specify a git url to clone workflow')
return False
try:
if self._git_branch:
Repo.clone_from(self._git,
self._path,
branch=self._git_branch,
config='http.sslVerify=false')
else:
Repo.clone_from(self._git,
self._path,
config='http.sslVerify=false')
except GitError as err:
Log.an().error('cannot clone git repo: %s [%s]', self._git,
str(err))
return False
return True
def apps(self, name=None):
"""
Get app dicts.
Return either all apps (name=None), or a specific app from dict. If
dict key doesn't exist, an empty dict is returned and an error is
logged.
Args:
name: name of app to return, None (default) indicates all apps.
Returns:
Dict of apps.
"""
if name is None:
return self._apps
if name not in self._apps:
Log.an().error('app not found: %s', name)
return {}
return self._apps[name]
def main():
"""
Geneflow CLI main entrypoint.
Args:
None.
Returns:
Nothing.
"""
args = parse_args()
if not args:
sys.exit(1)
# configure logging
Log.config(args.log_level, args.log_file)
# call the appropriate command
if not args.func(args):
sys.exit(1)
sys.exit(0)
def _copy_agave_local(src_uri, dest_uri, agave, local=None):
"""
Copy Agave data to a local destination using AgavePy Wrapper.
Args:
src_uri: Source URI parsed into dict with URIParser.
dest_uri: Destination URI parsed into dict with URIParser.
agave: dict that contains:
agave_wrapper: Agave wrapper object.
Returns:
On success: True.
On failure: False.
"""
if not agave['agave_wrapper'].files_download(src_uri['authority'],
src_uri['chopped_path'],
dest_uri['chopped_path'], -1):
Log.an().error('cannot copy from %s to %s', src_uri['uri'],
dest_uri['uri'])
return False
return True
def _copy_agave_agave(src_uri, dest_uri, agave):
"""
Copy Agave data using AgavePy Wrapper.
Args:
src_uri: Source URI parsed into dict with URIParser.
dest_uri: Destination URI parsed into dict with URIParser.
agave: dict that contains:
agave_wrapper: Agave wrapper object.
Returns:
On success: True.
On failure: False.
"""
if not agave['agave_wrapper'].files_import_from_agave(
dest_uri['authority'], dest_uri['folder'], dest_uri['name'],
src_uri['uri']):
Log.an().error('cannot copy from %s to %s', src_uri['uri'],
dest_uri['uri'])
return False
return True
def stage(self, **kwargs):
"""
Copy data to all contexts from source URI.
Set _staged indicator to True on success.
Args:
self: class instance.
**kwargs: additional arguments required by DataManager.copy().
Returns:
True or False.
"""
for context in self._parsed_data_uris:
if context != self._source_context:
if self._clean:
# remove target URI first
pass
if not DataManager.copy(
parsed_src_uri=self._parsed_data_uris\
[self._source_context],
parsed_dest_uri=self._parsed_data_uris[context],
**kwargs
):
msg = 'cannot stage data by copying from {} to {}'.format(
self._parsed_data_uris[self._source_context]\
['chopped_uri'],
self._parsed_data_uris[context]['chopped_uri']
)
Log.an().error(msg)
return self._fatal(msg)
self._staged = True
return True
def _get_map_uri_list(self):
"""
Get the contents of the map URI (local URI).
Args:
self: class instance.
Returns:
Array of base file names in the map URI. Returns False on
exception.
"""
combined_file_list = []
for uri in self._parsed_map_uris:
# make sure map URI is compatible scheme (local)
if uri['scheme'] != 'local':
msg = 'invalid map uri scheme for this step: {}'.format(
uri['scheme'])
Log.an().error(msg)
return self._fatal(msg)
# get file list from URI
file_list = DataManager.list(parsed_uri=uri,
globstr=self._step['map']['glob'])
if file_list is False:
msg = 'cannot get contents of map uri: {}'\
.format(uri['chopped_uri'])
Log.an().error(msg)
return self._fatal(msg)
for f in file_list:
combined_file_list.append({
'chopped_uri': uri['chopped_uri'],
'filename': f
})
return combined_file_list
def _agave_connect(self):
agave_connection_type = self._config['agave'].get(
'connection_type', 'impersonate')
if agave_connection_type == 'impersonate':
self._agave = Agave(api_server=self._config['agave']['server'],
username=self._config['agave']['username'],
password=self._config['agave']['password'],
token_username=self._job['username'],
client_name=self._config['agave']['client'],
api_key=self._config['agave']['key'],
api_secret=self._config['agave']['secret'],
verify=False)
# when using impersonate, token_username is taken from the job
# description and is used to access archived job data
self._config['agave']['token_username'] = self._job['username']
elif agave_connection_type == 'agave-cli':
# get credentials from ~/.agave/current
agave_clients = Agave._read_clients()
agave_clients[0]['verify'] = False # don't verify ssl
self._agave = Agave(**agave_clients[0])
# when using agave-cli, token_username must be the same as the
# stored creds in user's home directory, this can be different
# from job username
self._config['agave']['token_username'] \
= agave_clients[0]['username']
else:
Log.an().error('invalid agave connection type: %s',
agave_connection_type)
return False
return True
def _init_inputs(self):
"""
Initialize Graph input nodes by creating WorkflowInput instances.
Also populate the remaining contexts with URIs.
Args:
None.
Returns:
On failure: Raises WorkflowDAGException.
"""
for node_name in self._topo_sort:
node = self._graph.nodes[node_name]
if node['type'] == 'input':
# update source context URI
try:
node['contexts'][node['source_context']]\
= self._context_uris['inputs']\
[node['source_context']][node['name']]
except KeyError as err:
msg = 'invalid source context: {} [{}]'.format(
node['source_context'], str(err)
)
Log.an().error(msg)
raise WorkflowDAGException(msg) from err
# construct other context URIs
for context in node['contexts']:
if (
context != node['source_context']
and not node['contexts'][context]
):
try:
node['contexts'][context]\
= self._context_uris['inputs'][context]\
[node['name']]
except KeyError as err:
msg = 'invalid context: {} [{}]'.format(
context, str(err)
)
Log.an().error(msg)
raise WorkflowDAGException(msg) from err
# create instance of WorkflowInput class
node['node'] = WorkflowInput(
node['name'],
node['contexts'],
node['source_context']
)
if not node['node'].initialize():
msg = 'cannot initialize graph node: {}'.format(node_name)
Log.an().error(msg)
raise WorkflowDAGException(msg)
def migrate_db(args, other_args, subparser=None):
"""
Migrate SQL DB schema. Currently only works for MySQL databases.
Args:
args.config: GeneFlow config file path.
args.environment: Config environment.
Returns:
On success: True.
On failure: False.
"""
config = args.config
environment = args.environment
cfg = Config()
if not cfg.load(config):
Log.an().error('cannot load config file: %s', config)
return False
config_dict = cfg.config(environment)
if not config_dict:
Log.an().error('invalid config environment: %s', environment)
return False
if config_dict['database']['type'] != 'mysql':
Log.an().error('only mysql databases can be migrated')
return False
migrations_path = str(Path(GF_PACKAGE_PATH, 'data/migrations'))
try:
database = get_backend('{}://{}:{}@{}/{}'.format(
config_dict['database']['type'], config_dict['database']['user'],
config_dict['database']['password'],
config_dict['database']['host'],
config_dict['database']['database']))
migrations = read_migrations(migrations_path)
with database.lock():
database.apply_migrations(database.to_apply(migrations))
except Exception as err:
Log.an().error('cannot migrate database [%s]', str(err))
return False
return True
def _init_workflow_contexts(self):
"""
Import modules and load classes for each workflow context.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
# currently the union of all execution and data contexts will be used
# to initialize workflow contexts/classes. the reason is that all supported
# data contexts are also execution contexts. This may change in the future
# with data-only contexts (e.g., http/s). In that case, a new method
# (_init_data_contexts) will be added to populate a _data_context variable.
for context in self._exec_contexts | self._data_contexts:
mod_name = '{}_workflow'.format(context)
cls_name = '{}Workflow'.format(context.capitalize())
try:
workflow_mod = __import__(
'geneflow.extend.{}'.format(mod_name),
fromlist=[cls_name]
)
except ImportError as err:
msg = 'cannot import workflow module: {} [{}]'.format(
mod_name, str(err)
)
Log.an().error(msg)
return self._fatal(msg)
try:
workflow_class = getattr(workflow_mod, cls_name)
except AttributeError as err:
msg = 'cannot import workflow class: {} [{}]'.format(
cls_name, str(err)
)
Log.an().error(msg)
return self._fatal(msg)
self._workflow_context[context] = workflow_class(
self._config, self._job, self._parsed_job_work_uri
)
# perform context-specific init
if not self._workflow_context[context].initialize():
msg = (
'cannot initialize workflow context: {}'.format(cls_name)
)
Log.an().error(msg)
return self._fatal(msg)
return True
def _load_apps(self):
"""
Load and validate app definitions from the database.
Args:
self: class instance
Returns:
On success: True.
On failure: False.
"""
try:
data_source = DataSource(self._config['database'])
except DataSourceException as err:
msg = 'data source initialization error [{}]'.format(str(err))
Log.an().error(msg)
return self._fatal(msg)
self._apps = data_source.get_app_defs_by_workflow_id(
self._job['workflow_id']
)
if self._apps is False:
msg = 'cannot load apps from data source: workflow_id={}'.\
format(self._job['workflow_id'])
Log.an().error(msg)
return self._fatal(msg)
if not self._apps:
msg = 'no apps found for workflow: workflow_id={}'.\
format(self._job['workflow_id'])
Log.an().error(msg)
return self._fatal(msg)
# validate the app definitions
for app in self._apps:
valid_def = Definition.validate_app(self._apps[app])
if valid_def is False:
msg = 'invalid app definition:\n{}'\
.format(yaml.dump(self._apps[app]))
Log.an().error(msg)
return self._fatal(msg)
self._apps[app] = valid_def
return True
def jobs(self, name=None):
"""
Get job dicts.
Return either all jobs (name=None), or a specific job from dict. If
dict key doesn't exist, an empty dict is returned and an error is
logged.
Args:
name: name of job to return, None (default) indicates all
jobs.
Returns:
Dict of jobs.
"""
if name is None:
return self._jobs
if name not in self._jobs:
Log.an().error('job not found: %s', name)
return {}
return self._jobs[name]