def test_list_submissions(self):
"""Test list_submissions()."""
r = fapi.list_submissions(self.project, self.workspace)
print(r.status_code, r.content)
self.assertEqual(r.status_code, 200)
#print('SUBMISSION IS', z, z.json())
#z = fapi.get_config_template(namespace='dockstore', method='dockstore-tool-cosi2', version=1)
#print(z.json())
def _pretty_print_json(json_dict, sort_keys=True):
"""Return a pretty-printed version of a dict converted to json, as a string."""
return json.dumps(json_dict, indent=4, separators=(',', ': '), sort_keys=sort_keys)
def _write_json(fname, **json_dict):
dump_file(fname=fname, value=_pretty_print_json(json_dict))
#print('ENTITIES ARE', fapi.list_entity_types(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE).json())
z = fapi.list_submissions(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE)
#print('SUBMISSIONS ARE', z, z.json())
for s in sorted(list(z.json()), key=operator.itemgetter('submissionDate'), reverse=True)[:1]:
#if not s['submissionDate'].startswith('2020-06-29'): continue
print('====================================================')
print(s)
y = fapi.get_submission(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE, submission_id=s['submissionId']).json()
if 'workflowId' not in y['workflows'][0]: continue
zz = fapi.get_workflow_metadata(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE, submission_id=s['submissionId'],
workflow_id=y['workflows'][0]['workflowId']).json()
_write_json('tmp/j2.json', **zz)
dump_file(fname='tmp/w.wdl', value=zz['submittedFiles']['workflow'])
succ = [v["succeeded"] for v in zz['outputs']["run_sims_cosi2.replicaInfos"]]
print(f'Succeeded: {sum(succ)} of {len(succ)}')
def supervise_until_complete(monitor_data, dependencies, args, recovery_file):
""" Supervisor loop. Loop forever until all tasks are evaluated or completed """
project = args['project']
workspace = args['workspace']
namespace = args['namespace']
sample_sets = args['sample_sets']
recovery_data = {'args': args}
if not validate_monitor_tasks(dependencies, args):
logging.error("Errors found, aborting...")
return
while True:
# There are 4 possible states for each node:
# 1. Not Started -- In this state, check all the dependencies for the
# node (possibly 0). If all of them have been evaluated, and the
# satisfiedMode is met, start the task, change to "Running". if
# satisfiedMode is not met, change to "Evaluated"
#
# 2. Running -- Submitted in FC. Check the submission endpoint, and
# if it has completed, change to "Completed", set evaluated=True,
# and whether the task succeeded
# Otherwise, do nothing
#
# 3. Completed -- Job ran in FC and either succeeded or failed. Do nothing
# 4. Evaluated -- All dependencies evaluated, but this task did not run
# do nothing
# Keep a tab of the number of jobs in each category
running = 0
waiting = 0
completed = 0
# Get the submissions
r = fapi.list_submissions(project, workspace)
sub_list = r.json()
#TODO: filter this list by submission time first?
sub_lookup = {s["submissionId"]: s for s in sub_list}
# Keys of dependencies is the list of tasks to run
for n in dependencies:
for sset in sample_sets:
task_data = monitor_data[n][sset]
if task_data['state'] == "Not Started":
# See if all of the dependencies have been evaluated
upstream_evaluated = True
for dep in dependencies[n]:
# Look up the status of the task
upstream_task_data = monitor_data[
dep['upstream_task']][sset]
if not upstream_task_data.get('evaluated'):
upstream_evaluated = False
# if all of the dependencies have been evaluated, we can evaluate
# this node
if upstream_evaluated:
# Now check the satisfied Mode of all the dependencies
should_run = True
for dep in dependencies[n]:
upstream_task_data = monitor_data[
dep['upstream_task']][sset]
mode = dep['satisfiedMode']
# Task must have succeeded for OnComplete
if mode == '"OnComplete"' and not upstream_task_data[
'succeeded']:
should_run = False
# 'Always' and 'Optional' run once the deps have been
# evaluated
if should_run:
# Submit the workflow to FC
fc_config = n
logging.info("Starting workflow " + fc_config +
" on " + sset)
# How to handle errors at this step?
for retry in range(3):
r = fapi.create_submission(project,
workspace,
namespace,
fc_config,
sset,
etype="sample_set",
expression=None)
if r.status_code == 201:
task_data['submissionId'] = r.json(
)['submissionId']
task_data['state'] = "Running"
running += 1
break
else:
# There was an error, under certain circumstances retry
logging.debug(
"Create_submission for " + fc_config +
"failed on " + sset +
" with the following response:" +
r.content + "\nRetrying...")
else:
# None of the attempts above succeeded, log an error, mark as failed
logging.error("Maximum retries exceeded")
task_data['state'] = 'Completed'
task_data['evaluated'] = True
task_data['succeeded'] = False
else:
# This task will never be able to run, mark evaluated
task_data['state'] = "Evaluated"
task_data['evaluated'] = True
completed += 1
else:
waiting += 1
elif task_data['state'] == "Running":
submission = sub_lookup[task_data['submissionId']]
status = submission['status']
if status == "Done":
# Look at the individual workflows to see if there were
# failures
logging.info("Workflow " + n + " completed for " +
sset)
success = 'Failed' not in submission[
'workflowStatuses']
task_data['evaluated'] = True
task_data['succeeded'] = success
task_data['state'] = "Completed"
completed += 1
else:
# Submission isn't done, don't do anything
running += 1
else:
# Either Completed or evaluated
completed += 1
# Save the state of the monitor for recovery purposes
# Have to do this for every workflow + sample_set so we don't lose track of any
recovery_data['monitor_data'] = monitor_data
recovery_data['dependencies'] = dependencies
with open(recovery_file, 'w') as rf:
json.dump(recovery_data, rf)
logging.info("{0} Waiting, {1} Running, {2} Completed".format(
waiting, running, completed))
# If all tasks have been evaluated, we are done
if all(monitor_data[n][sset]['evaluated'] for n in monitor_data
for sset in monitor_data[n]):
logging.info("DONE.")
break
time.sleep(30)