def test_list_submissions(self): """Test list_submissions().""" r = fapi.list_submissions(self.project, self.workspace) print(r.status_code, r.content) self.assertEqual(r.status_code, 200)
#print('SUBMISSION IS', z, z.json()) #z = fapi.get_config_template(namespace='dockstore', method='dockstore-tool-cosi2', version=1) #print(z.json()) def _pretty_print_json(json_dict, sort_keys=True): """Return a pretty-printed version of a dict converted to json, as a string.""" return json.dumps(json_dict, indent=4, separators=(',', ': '), sort_keys=sort_keys) def _write_json(fname, **json_dict): dump_file(fname=fname, value=_pretty_print_json(json_dict)) #print('ENTITIES ARE', fapi.list_entity_types(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE).json()) z = fapi.list_submissions(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE) #print('SUBMISSIONS ARE', z, z.json()) for s in sorted(list(z.json()), key=operator.itemgetter('submissionDate'), reverse=True)[:1]: #if not s['submissionDate'].startswith('2020-06-29'): continue print('====================================================') print(s) y = fapi.get_submission(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE, submission_id=s['submissionId']).json() if 'workflowId' not in y['workflows'][0]: continue zz = fapi.get_workflow_metadata(namespace=SEL_NAMESPACE, workspace=SEL_WORKSPACE, submission_id=s['submissionId'], workflow_id=y['workflows'][0]['workflowId']).json() _write_json('tmp/j2.json', **zz) dump_file(fname='tmp/w.wdl', value=zz['submittedFiles']['workflow']) succ = [v["succeeded"] for v in zz['outputs']["run_sims_cosi2.replicaInfos"]] print(f'Succeeded: {sum(succ)} of {len(succ)}')
def supervise_until_complete(monitor_data, dependencies, args, recovery_file): """ Supervisor loop. Loop forever until all tasks are evaluated or completed """ project = args['project'] workspace = args['workspace'] namespace = args['namespace'] sample_sets = args['sample_sets'] recovery_data = {'args': args} if not validate_monitor_tasks(dependencies, args): logging.error("Errors found, aborting...") return while True: # There are 4 possible states for each node: # 1. Not Started -- In this state, check all the dependencies for the # node (possibly 0). If all of them have been evaluated, and the # satisfiedMode is met, start the task, change to "Running". if # satisfiedMode is not met, change to "Evaluated" # # 2. Running -- Submitted in FC. Check the submission endpoint, and # if it has completed, change to "Completed", set evaluated=True, # and whether the task succeeded # Otherwise, do nothing # # 3. Completed -- Job ran in FC and either succeeded or failed. Do nothing # 4. Evaluated -- All dependencies evaluated, but this task did not run # do nothing # Keep a tab of the number of jobs in each category running = 0 waiting = 0 completed = 0 # Get the submissions r = fapi.list_submissions(project, workspace) sub_list = r.json() #TODO: filter this list by submission time first? sub_lookup = {s["submissionId"]: s for s in sub_list} # Keys of dependencies is the list of tasks to run for n in dependencies: for sset in sample_sets: task_data = monitor_data[n][sset] if task_data['state'] == "Not Started": # See if all of the dependencies have been evaluated upstream_evaluated = True for dep in dependencies[n]: # Look up the status of the task upstream_task_data = monitor_data[ dep['upstream_task']][sset] if not upstream_task_data.get('evaluated'): upstream_evaluated = False # if all of the dependencies have been evaluated, we can evaluate # this node if upstream_evaluated: # Now check the satisfied Mode of all the dependencies should_run = True for dep in dependencies[n]: upstream_task_data = monitor_data[ dep['upstream_task']][sset] mode = dep['satisfiedMode'] # Task must have succeeded for OnComplete if mode == '"OnComplete"' and not upstream_task_data[ 'succeeded']: should_run = False # 'Always' and 'Optional' run once the deps have been # evaluated if should_run: # Submit the workflow to FC fc_config = n logging.info("Starting workflow " + fc_config + " on " + sset) # How to handle errors at this step? for retry in range(3): r = fapi.create_submission(project, workspace, namespace, fc_config, sset, etype="sample_set", expression=None) if r.status_code == 201: task_data['submissionId'] = r.json( )['submissionId'] task_data['state'] = "Running" running += 1 break else: # There was an error, under certain circumstances retry logging.debug( "Create_submission for " + fc_config + "failed on " + sset + " with the following response:" + r.content + "\nRetrying...") else: # None of the attempts above succeeded, log an error, mark as failed logging.error("Maximum retries exceeded") task_data['state'] = 'Completed' task_data['evaluated'] = True task_data['succeeded'] = False else: # This task will never be able to run, mark evaluated task_data['state'] = "Evaluated" task_data['evaluated'] = True completed += 1 else: waiting += 1 elif task_data['state'] == "Running": submission = sub_lookup[task_data['submissionId']] status = submission['status'] if status == "Done": # Look at the individual workflows to see if there were # failures logging.info("Workflow " + n + " completed for " + sset) success = 'Failed' not in submission[ 'workflowStatuses'] task_data['evaluated'] = True task_data['succeeded'] = success task_data['state'] = "Completed" completed += 1 else: # Submission isn't done, don't do anything running += 1 else: # Either Completed or evaluated completed += 1 # Save the state of the monitor for recovery purposes # Have to do this for every workflow + sample_set so we don't lose track of any recovery_data['monitor_data'] = monitor_data recovery_data['dependencies'] = dependencies with open(recovery_file, 'w') as rf: json.dump(recovery_data, rf) logging.info("{0} Waiting, {1} Running, {2} Completed".format( waiting, running, completed)) # If all tasks have been evaluated, we are done if all(monitor_data[n][sset]['evaluated'] for n in monitor_data for sset in monitor_data[n]): logging.info("DONE.") break time.sleep(30)