def test_abort(self, mock_request):
workflow_id = "01234"
expected = {"id": workflow_id, "status": "Aborting"}
def _request_callback(request, context):
context.status_code = 200
context.headers['test'] = 'header'
return expected
for cromwell_auth in self.auth_options:
mock_request.post(
cromwell_auth.url + '/api/workflows/v1/{}/abort'.format(workflow_id),
json=_request_callback,
)
result = CromwellAPI.abort(workflow_id, cromwell_auth)
self.assertEqual(result.json(), expected)
def main():
# Configure argparser
argparser = get_argparser()
# Parse the arguments
args = argparser.parse_args()
# Input files: json input file to be used as template and
batch_input_json = args.input_json
batch_label_json = args.label_json
wdl_workflow = args.wdl_workflow
wdl_imports = args.wdl_imports
wdl_options = args.wdl_options
output_dir = args.output_dir
batch_conflict_action = args.batch_conflict_action
cromwell_url = args.cromwell_url
# Standardize url
cromwell_url = utils.fix_url(cromwell_url)
# Configure logging appropriate for verbosity
utils.configure_logging(args.verbosity_level)
# Read in batch inputs
with open(batch_input_json, "r") as fh:
batch_inputs = json.load(fh)
with open(batch_label_json, "r") as fh:
batch_labels = json.load(fh)
# Convert inputs/lables to lists if they're not already lists
batch_inputs = [batch_inputs
] if not isinstance(batch_inputs, list) else batch_inputs
batch_labels = [batch_labels
] if not isinstance(batch_labels, list) else batch_labels
assert len(batch_inputs) == len(
batch_labels), "Batch label and input files are different sizes!"
# Check to make sure all workflow labels have required label keys
for wf_labels in batch_labels:
wdl.validate_wf_labels(wf_labels)
# Authenticate and validate cromwell server
auth = cromwell.get_cromwell_auth(url=cromwell_url)
cromwell.validate_cromwell_server(auth)
# Create a report to detail what jobs were run/skipped/failed
job_report = copy.deepcopy(batch_labels)
batch_name = batch_labels[0][const.CROMWELL_BATCH_LABEL]
report_file = "{0}/{1}.submit_batch.{2}.xlsx".format(
output_dir, batch_name, time.strftime("%Y%m%d-%H%M%S"))
# Loop through workflows to see if they need to be run/rerun
submitted_wfs = 0
for i in range(len(batch_inputs)):
# Get inputs, labels, and batch_sample label for next workflow in batch
wf_input = batch_inputs[i]
wf_labels = batch_labels[i]
batch_sample_label = wf_labels[const.CROMWELL_BATCH_SAMPLE_LABEL]
# Try to run the workflow
try:
# Get list of previous workflows in this batch with the same sample name (batch conflicts)
batch_conflict_wfs = cromwell.get_batch_conflicts(
auth, batch_sample_label)
# Determine how to resolve each batch conflict
can_submit_wf = True
for batch_conflict_wf in batch_conflict_wfs:
can_overide_conflict_wf, abort_conflict_wf = resolve_batch_conflict(
auth, batch_conflict_wf, batch_conflict_action)
# Exclude prior conflicting wf from the current active batch
if can_overide_conflict_wf:
cromwell.update_wf_batch_status(auth,
batch_conflict_wf,
include_in_batch=False)
# Abort prior conflicting wf if in non-terminal state (fail, success)
if abort_conflict_wf:
CromwellAPI.abort(batch_conflict_wf,
auth,
raise_for_status=True)
logging.warning(
"Aborted conflicting wf '{0}' "
"with duplicate batch_sample_id '{1}'".format(
batch_conflict_wf, batch_sample_label))
# Workflow can only be submitted if it can override all prior workflows
can_submit_wf = can_submit_wf and can_overide_conflict_wf
# Workflow id of submission and message to print in job report file.
# Will be overwritten if job is submitted.
wf_id = "Not submitted"
msg = "SupercededWF is either running/submitted or was successful."
# Run the workflow if you can
if can_submit_wf:
# Show some logging stuff
logging.info(
"Submitting workflow for '{0}'".format(batch_sample_label))
if batch_conflict_wfs:
logging.warning("Superceded workflows: {0}".format(
", ".join(batch_conflict_wfs)))
# Submit workflow and get id of the newly submitted workflow
wf_id = cromwell.submit_wf_from_dict(auth,
wdl_workflow,
input_dict=wf_input,
dependencies=wdl_imports,
label_dict=wf_labels,
options_file=wdl_options)
# Increment counter of successfully submitted workflows
submitted_wfs += 1
msg = ""
# Update information for report
job_report[i][const.CROMWELL_WF_ID_FIELD] = wf_id
job_report[i][const.SUPERCEDED_WF_FIELD] = ", ".join(
batch_conflict_wfs)
job_report[i][const.REPORT_INFO_FIELD] = msg
except BaseException:
# Log any successful submissions and indicate workflow which caused failure
logging.info(
"Successfully submitted {0} out of {1} workflows in batch!".
format(submitted_wfs, len(batch_inputs)))
logging.info("Writing workflow report...")
output_job_report(job_report, report_file, failed_workflow_index=i)
raise
# Log submission if all workflows submitted successfully
logging.info(
"Successfully submitted {0} out of {1} workflows in batch!".format(
submitted_wfs, len(batch_inputs)))
logging.info("Writing workflow report...")
output_job_report(job_report, report_file)
def main():
# Configure argparser
argparser = get_argparser()
# Parse the arguments
args = argparser.parse_args()
# Input files: json input file to be used as template and
batch_name = args.batch_name
cromwell_url = args.cromwell_url
# Standardize url
cromwell_url = utils.fix_url(cromwell_url)
# Configure logging appropriate for verbosity
utils.configure_logging(args.verbosity_level)
# Authenticate and validate cromwell server
auth = cromwell.get_cromwell_auth(url=cromwell_url)
cromwell.validate_cromwell_server(auth)
# Otherwise just grab all of the workflows with batch-name
batch_wfs = cromwell.query_workflows(
auth, {"label": {
const.CROMWELL_BATCH_LABEL: batch_name
}})
# Error out if batch doesn't actually exist
if not batch_wfs:
logging.error(
"No batch exists on current cromwell server with batch-name '{0}'".
format(batch_name))
raise IOError
# Terminal wf status codes
terminal_states = [
const.CROMWELL_ABORTING_STATUS, const.CROMWELL_ABORTED_STATUS,
const.CROMWELL_SUCCESS_STATUS, const.CROMWELL_FAILED_STATUS
]
logging.info("Aborting workflows...")
aborted_wfs = 0
running_wfs = 0
for wf in batch_wfs:
wf_status = cromwell.get_wf_status(auth, wf)
if wf_status not in terminal_states:
try:
logging.info("Aborting wf: {0}".format(wf))
CromwellAPI.abort(wf, auth, raise_for_status=True)
aborted_wfs += 1
except requests.exceptions.HTTPError:
logging.warning(
"Unable to abort wf '{0}' for some reason...".format(wf))
finally:
running_wfs += 1
success_rate = 0.0 if running_wfs == 0 else (aborted_wfs /
(1.0 * running_wfs)) * 100
logging.info(
"{0}/{1} ({2}%) pending batch workflows successfully aborted!".format(
aborted_wfs, running_wfs, success_rate))
