def submit_wf_from_dict(cromwell_auth,
wdl_workflow,
input_dict,
dependencies=None,
label_dict=None,
options_file=None):
# Write input and label files to tmp files
input_file = tempfile.NamedTemporaryFile()
with open(input_file.name, "w") as fh:
json.dump(input_dict, fh)
if label_dict:
label_file = tempfile.NamedTemporaryFile()
with open(label_file.name, "w") as fh:
json.dump(label_dict, fh)
else:
label_file = None
try:
# Submit workflow and return id
result = CromwellAPI.submit(cromwell_auth,
wdl_workflow,
input_file.name,
dependencies=dependencies,
label_file=label_file.name,
options_file=options_file,
raise_for_status=True)
return result.json()["id"]
finally:
# Close temp files no matter what
input_file.close()
if label_dict:
label_file.close()
def _submit_workflows(self, cromwell_auth, mock_request,
_request_callback):
mock_request.post(cromwell_auth.url + '/api/workflows/v1',
json=_request_callback)
return CromwellAPI.submit(
auth=cromwell_auth,
wdl_file=self.wdl_file,
inputs_files=self.inputs_file,
options_file=self.options_file,
dependencies=self.zip_file,
label_file=self.label,
)
def submit_jobs(self):
resp_list = []
for _index, _args_dict in self.final_args_dict.items():
new_inputs_dict = deepcopy(self.inputs_dict)
new_inputs_dict[
"genericworkflow.GenericTask.shell_command"] = _args_dict[
'command']
new_inputs_dict[
"genericworkflow.GenericTask.input_files"] = _args_dict[
'remote_input_files']
temp_resp = cwt.submit(auth=self.auth_obj,
wdl_file=io.BytesIO(self.wdl_text.encode()),
inputs_files=io.BytesIO(
json.dumps(new_inputs_dict).encode()),
options_file=io.BytesIO(
json.dumps(self.options_dict).encode()))
resp_list.append(temp_resp.json()['id'])
#resp_list.append(new_inputs_dict)
self.resp_list = resp_list
return (self.resp_list)
def calculate_metric(template_values, scan_values):
merged_values = {
k: v
for (k, v) in (template_values.items() + scan_values.items())
}
merged_json_file, merged_json_path = tempfile.mkstemp()
with open(merged_json_path, 'w') as f:
json.dump(merged_values, f)
cromwell_auth = CromwellAuth(url=args.cromwell_server,
header={'Authorization': 'bearer fake_token'},
auth=None)
with open(args.workflow_wdl, 'r') as w, open(merged_json_path, 'r') as j:
submit = CromwellAPI.submit(cromwell_auth, w, j)
workflow_id = submit.json()['id']
logger.info('Submitted workflow: ' + workflow_id)
time.sleep(5)
logger.info('Waiting for workflow to complete...')
# Query workflow status indefinitely until success or failure returned.
# If success returned, attempt to retrieve objective_value from metadata and return.
# If failure returned or if exception raised during metadata retreival, return bad_value.
try:
while True:
try:
CromwellAPI.wait([workflow_id],
cromwell_auth,
timeout_minutes=600,
poll_interval_seconds=20,
verbose=False)
response = CromwellAPI.status(workflow_id, cromwell_auth)
status = response.json()['status']
if status == 'Succeeded':
logger.info('Workflow succeeded...')
break
except WorkflowFailedException:
logger.info('Workflow failed, returning bad value...')
return bad_value
except Exception as e:
logger.info(e)
logger.info(
'Cromwell exception, retrying wait and status check...')
logger.info('Getting metadata...')
session = retry_session(retries=10)
metadata = session.post(
url=cromwell_auth.url +
CromwellAPI._metadata_endpoint.format(uuid=workflow_id),
auth=cromwell_auth.auth,
headers=cromwell_auth.header)
workflow_name = metadata.json()['workflowName']
objective_value = metadata.json()['outputs'][
'{}.objective_value'.format(workflow_name)]
return objective_value
except Exception as e:
logger.info(e)
logger.info(
'Cromwell exception during metadata retrieval, returning bad value...'
)
return bad_value
"final_workflow_outputs_dir": output_base_location,
"use_relative_output_paths": True,
"final_call_logs_dir": "{}/call_logs".format(output_base_location),
"jes_gcs_root": cromwell_runs_bucket,
"google_labels": {
"pipeline-name": "gatk4-germline-snps-indels",
"project-name": "comparing-gatk-sentieon-dragen"
}
}
# -
input_iobytes = io.BytesIO(json.dumps(jj_input_json).encode())
options_iobytes = io.BytesIO(json.dumps(jj_options_dict).encode())
jj_resp = cwt.submit(auth_obj,
wdl_file = cromwell_functions.get_wdl_iobytes("gs://bioskryb_dev_wdl_and_inputs/gatk-workflows/gatk4-germline-snps-indels/2.0.0/JointGenotyping.wdl", storage_client),
inputs_files = input_iobytes,
options_file = options_iobytes
)
jj_resp.content
gvcf_filenames_n16
# ## Sentieon - gVCF generation
# The same 16 samples from above (plus the reference) are joint genotyped using Sentieon
# First, generate gVCFs
r1_fastq_files = !gsutil ls gs://bioskryb-vumc-data/Nova181_H2VMKDSXY/*_R1_*fastq.gz
r2_fastq_files = !gsutil ls gs://bioskryb-vumc-data/Nova182_H2VGNDSXY/*_R1_*fastq.gz