def test_abort(self, mock_request): workflow_id = "01234" expected = {"id": workflow_id, "status": "Aborting"} def _request_callback(request, context): context.status_code = 200 context.headers['test'] = 'header' return expected for cromwell_auth in self.auth_options: mock_request.post( cromwell_auth.url + '/api/workflows/v1/{}/abort'.format(workflow_id), json=_request_callback, ) result = CromwellAPI.abort(workflow_id, cromwell_auth) self.assertEqual(result.json(), expected)
def main(): # Configure argparser argparser = get_argparser() # Parse the arguments args = argparser.parse_args() # Input files: json input file to be used as template and batch_input_json = args.input_json batch_label_json = args.label_json wdl_workflow = args.wdl_workflow wdl_imports = args.wdl_imports wdl_options = args.wdl_options output_dir = args.output_dir batch_conflict_action = args.batch_conflict_action cromwell_url = args.cromwell_url # Standardize url cromwell_url = utils.fix_url(cromwell_url) # Configure logging appropriate for verbosity utils.configure_logging(args.verbosity_level) # Read in batch inputs with open(batch_input_json, "r") as fh: batch_inputs = json.load(fh) with open(batch_label_json, "r") as fh: batch_labels = json.load(fh) # Convert inputs/lables to lists if they're not already lists batch_inputs = [batch_inputs ] if not isinstance(batch_inputs, list) else batch_inputs batch_labels = [batch_labels ] if not isinstance(batch_labels, list) else batch_labels assert len(batch_inputs) == len( batch_labels), "Batch label and input files are different sizes!" # Check to make sure all workflow labels have required label keys for wf_labels in batch_labels: wdl.validate_wf_labels(wf_labels) # Authenticate and validate cromwell server auth = cromwell.get_cromwell_auth(url=cromwell_url) cromwell.validate_cromwell_server(auth) # Create a report to detail what jobs were run/skipped/failed job_report = copy.deepcopy(batch_labels) batch_name = batch_labels[0][const.CROMWELL_BATCH_LABEL] report_file = "{0}/{1}.submit_batch.{2}.xlsx".format( output_dir, batch_name, time.strftime("%Y%m%d-%H%M%S")) # Loop through workflows to see if they need to be run/rerun submitted_wfs = 0 for i in range(len(batch_inputs)): # Get inputs, labels, and batch_sample label for next workflow in batch wf_input = batch_inputs[i] wf_labels = batch_labels[i] batch_sample_label = wf_labels[const.CROMWELL_BATCH_SAMPLE_LABEL] # Try to run the workflow try: # Get list of previous workflows in this batch with the same sample name (batch conflicts) batch_conflict_wfs = cromwell.get_batch_conflicts( auth, batch_sample_label) # Determine how to resolve each batch conflict can_submit_wf = True for batch_conflict_wf in batch_conflict_wfs: can_overide_conflict_wf, abort_conflict_wf = resolve_batch_conflict( auth, batch_conflict_wf, batch_conflict_action) # Exclude prior conflicting wf from the current active batch if can_overide_conflict_wf: cromwell.update_wf_batch_status(auth, batch_conflict_wf, include_in_batch=False) # Abort prior conflicting wf if in non-terminal state (fail, success) if abort_conflict_wf: CromwellAPI.abort(batch_conflict_wf, auth, raise_for_status=True) logging.warning( "Aborted conflicting wf '{0}' " "with duplicate batch_sample_id '{1}'".format( batch_conflict_wf, batch_sample_label)) # Workflow can only be submitted if it can override all prior workflows can_submit_wf = can_submit_wf and can_overide_conflict_wf # Workflow id of submission and message to print in job report file. # Will be overwritten if job is submitted. wf_id = "Not submitted" msg = "SupercededWF is either running/submitted or was successful." # Run the workflow if you can if can_submit_wf: # Show some logging stuff logging.info( "Submitting workflow for '{0}'".format(batch_sample_label)) if batch_conflict_wfs: logging.warning("Superceded workflows: {0}".format( ", ".join(batch_conflict_wfs))) # Submit workflow and get id of the newly submitted workflow wf_id = cromwell.submit_wf_from_dict(auth, wdl_workflow, input_dict=wf_input, dependencies=wdl_imports, label_dict=wf_labels, options_file=wdl_options) # Increment counter of successfully submitted workflows submitted_wfs += 1 msg = "" # Update information for report job_report[i][const.CROMWELL_WF_ID_FIELD] = wf_id job_report[i][const.SUPERCEDED_WF_FIELD] = ", ".join( batch_conflict_wfs) job_report[i][const.REPORT_INFO_FIELD] = msg except BaseException: # Log any successful submissions and indicate workflow which caused failure logging.info( "Successfully submitted {0} out of {1} workflows in batch!". format(submitted_wfs, len(batch_inputs))) logging.info("Writing workflow report...") output_job_report(job_report, report_file, failed_workflow_index=i) raise # Log submission if all workflows submitted successfully logging.info( "Successfully submitted {0} out of {1} workflows in batch!".format( submitted_wfs, len(batch_inputs))) logging.info("Writing workflow report...") output_job_report(job_report, report_file)
def main(): # Configure argparser argparser = get_argparser() # Parse the arguments args = argparser.parse_args() # Input files: json input file to be used as template and batch_name = args.batch_name cromwell_url = args.cromwell_url # Standardize url cromwell_url = utils.fix_url(cromwell_url) # Configure logging appropriate for verbosity utils.configure_logging(args.verbosity_level) # Authenticate and validate cromwell server auth = cromwell.get_cromwell_auth(url=cromwell_url) cromwell.validate_cromwell_server(auth) # Otherwise just grab all of the workflows with batch-name batch_wfs = cromwell.query_workflows( auth, {"label": { const.CROMWELL_BATCH_LABEL: batch_name }}) # Error out if batch doesn't actually exist if not batch_wfs: logging.error( "No batch exists on current cromwell server with batch-name '{0}'". format(batch_name)) raise IOError # Terminal wf status codes terminal_states = [ const.CROMWELL_ABORTING_STATUS, const.CROMWELL_ABORTED_STATUS, const.CROMWELL_SUCCESS_STATUS, const.CROMWELL_FAILED_STATUS ] logging.info("Aborting workflows...") aborted_wfs = 0 running_wfs = 0 for wf in batch_wfs: wf_status = cromwell.get_wf_status(auth, wf) if wf_status not in terminal_states: try: logging.info("Aborting wf: {0}".format(wf)) CromwellAPI.abort(wf, auth, raise_for_status=True) aborted_wfs += 1 except requests.exceptions.HTTPError: logging.warning( "Unable to abort wf '{0}' for some reason...".format(wf)) finally: running_wfs += 1 success_rate = 0.0 if running_wfs == 0 else (aborted_wfs / (1.0 * running_wfs)) * 100 logging.info( "{0}/{1} ({2}%) pending batch workflows successfully aborted!".format( aborted_wfs, running_wfs, success_rate))