def services_w_wfs_left2run():
services = []
for wf_service in wes_config():
received_submissions = get_submissions(wf_service, status='RECEIVED')
if received_submissions:
services.append(wf_service)
return services
def run_submission(eval_id, submission_id, wes_id='local'):
"""
For a single submission to a single evaluation queue, run
the workflow in a single environment.
"""
submission = eval.get_submission_bundle(eval_id, submission_id)
if submission['wes_id'] is not None:
wes_id = submission['wes_id']
logger.info(
"Submitting job '{}' for eval '{}' to WES endpoint '{}'".format(
submission_id, eval_id, wes_id))
client = WESClient(**config.wes_config()[wes_id])
run_data = client.run_workflow(submission['data'])
run_data['start_time'] = dt.datetime.now().ctime()
eval.update_submission_run(eval_id, submission_id, run_data)
eval.update_submission_status(eval_id, submission_id, 'SUBMITTED')
return eval.get_submission_bundle(eval_id, submission_id)
def monitor(submissions):
"""
Monitor progress of workflow jobs.
"""
current = dt.datetime.now()
statuses = []
for submission_dict in submissions:
status_dict = {}
for eval_id in submission_dict:
for submission_id, bundle in submission_dict[eval_id].items():
client = WESClient(**config.wes_config()[bundle['wes_id']])
run = client.get_workflow_run_status(bundle['run_id'])
if run['state'] in ['QUEUED', 'INITIALIZING', 'RUNNING']:
etime = util.convert_timedelta(
current - util.ctime2datetime(bundle['start_time']))
elif 'elapsed_time' not in bundle:
etime = 0
else:
etime = bundle['elapsed_time']
status_dict.setdefault(eval_id, {})[submission_id] = {
'queue_id': eval_id,
'job': bundle['job'],
'submission_status': bundle['submission_status'],
'wes_id': bundle['wes_id'],
'run_id': run['workflow_id'],
'run_status': run['state'],
'start_time': bundle['start_time'],
'elapsed_time': etime
}
statuses.append(status_dict)
status_df = pd.DataFrame.from_dict(
{(i, j): status[i][j]
for status in statuses for i in status.keys()
for j in status[i].keys()},
orient='index')
clear_output(wait=True)
display(status_df)
sys.stdout.flush()
if any(status_df['run_status'].isin(['QUEUED', 'INITIALIZING',
'RUNNING'])):
time.sleep(1)
monitor(statuses)
else:
print("Done")
def run_submission(wes_id, submission_id):
"""
For a single submission to a single evaluation queue, run
the workflow in a single environment.
"""
submission = get_submission_bundle(wes_id, submission_id)
logger.info(" Submitting to WES endpoint '{}':"
" \n - submission ID: {}".format(wes_id, submission_id))
client = WESClient(wes_config()[wes_id])
run_data = client.run(submission['data']['wf'],
submission['data']['jsonyaml'],
submission['data']['attachments'])
run_data['start_time'] = dt.datetime.now().ctime()
update_submission(wes_id, submission_id, 'run', run_data)
update_submission(wes_id, submission_id, 'status', 'SUBMITTED')
return run_data
def monitor_service(wf_service):
"""
Returns a dictionary of all of the jobs under a single wes service appropriate
for displaying as a pandas dataframe.
:param wf_service:
:return:
"""
status_dict = {}
submissions = get_json(queue_path())
for run_id in submissions[wf_service]:
sample_name = submissions[wf_service][run_id]['sample']
if 'run' not in submissions[wf_service][run_id]:
status_dict.setdefault(wf_service, {})[run_id] = {
'wf_id': submissions[wf_service][run_id]['wf_id'],
'run_id': '-',
'sample_name': sample_name,
'run_status': 'QUEUED',
'start_time': '-',
'elapsed_time': '-'
}
else:
if submissions[wf_service][run_id]['status'] in [
'COMPLETE', 'SYSTEM_ERROR', 'EXECUTOR_ERROR'
]:
run = submissions[wf_service][run_id]['run']
try:
wf_id = run['workflow_id']
except KeyError:
wf_id = run['run_id']
status_dict.setdefault(wf_service, {})[run_id] = {
'wf_id': submissions[wf_service][run_id]['wf_id'],
'run_id': wf_id,
'sample_name': sample_name,
'run_status': submissions[wf_service][run_id]['status'],
'start_time': run['start_time'],
'elapsed_time': run['elapsed_time']
}
else:
try:
run = submissions[wf_service][run_id]['run']
if 'run_id' not in run and 'workflow_id' not in run:
status_dict.setdefault(wf_service, {})[run_id] = {
'wf_id': submissions[wf_service][run_id]['wf_id'],
'run_id': '-',
'sample_name': sample_name,
'run_status': 'INITIALIZING',
'start_time': '-',
'elapsed_time': '-'
}
else:
client = WESClient(wes_config()[wf_service])
try:
wf_id = run['workflow_id']
except KeyError:
wf_id = run['run_id']
if 'state' not in run:
run['state'] = client.get_run_status(
wf_id)['state'].upper()
elif run['state'].upper() not in [
'COMPLETED', 'OK', 'EXECUTOR_ERROR',
'SYSTEM_ERROR'
]:
run['state'] = client.get_run_status(
wf_id)['state'].upper()
if run['state'] in [
'QUEUED', 'INITIALIZING', 'RUNNING'
]:
etime = convert_timedelta(
dt.datetime.now() -
ctime2datetime(run['start_time']))
elif 'elapsed_time' not in run:
etime = '0h:0m:0s'
else:
update_submission(wf_service, run_id, 'status',
run['state'])
etime = run['elapsed_time']
update_submission_run(wf_service, run_id,
'elapsed_time', etime)
status_dict.setdefault(wf_service, {})[run_id] = {
'wf_id': submissions[wf_service][run_id]['wf_id'],
'run_id': wf_id,
'sample_name': sample_name,
'run_status': run['state'],
'start_time': run['start_time'],
'elapsed_time': etime
}
except ConnectionError:
status_dict.setdefault(wf_service, {})[run_id] = {
'wf_id': 'ConnectionError',
'run_id': '-',
'sample_name': sample_name,
'run_status': '-',
'start_time': '-',
'elapsed_time': '-'
}
return status_dict