def copy_workflow_metadata(self, metadata, metadata_filename):
logging.info("Copying run metadata to %s", self.args.output_dir)
# Copy the run metadata
with open(metadata_filename, 'w') as f:
json.dump(metadata, f)
file_util.gsutil_cp([metadata_filename], "%s/" % self.args.output_dir)
def copy_workflow_output(self, result):
output_files = wdl_outputs_util.get_workflow_output(
result['outputs'], self.args.working_dir)
# Copy final output files (if any)
logging.info("Workflow output files = %s", output_files)
if output_files:
file_util.gsutil_cp(output_files, "%s/" % self.args.output_dir)
def copy_workflow_metadata(self, metadata, metadata_filename):
logging.info("Copying run metadata to %s", self.args.output_dir)
# Copy the run metadata
with open(metadata_filename, 'w') as f:
json.dump(metadata, f)
file_util.gsutil_cp([metadata_filename], "%s/" % self.args.output_dir)
def copy_workflow_output(self, result):
output_files = wdl_outputs_util.get_workflow_output(
result['outputs'], self.args.working_dir)
# Copy final output files (if any)
logging.info("Workflow output files = %s", output_files)
if output_files:
file_util.gsutil_cp(output_files, "%s/" % self.args.output_dir)
def run(self):
if self.args.batch is not None:
try:
file_util.gsutil_cp([os.environ['SUBMISSION_DATA_PATH']],
'submission.json')
with open('submission.json') as r:
submission_data = json.load(r)
runtime = submission_data[
'runtime'] if 'runtime' in submission_data else None
self.driver.start(
runtime['memory'] if runtime is not None else 3)
logging.info("Starting batch request")
# logging.info("SUBMITTING JOB " + repr((
# self.args.batch,
# self.args.wdl,
# self.args.workflow_inputs,
# self.args.workflow_options,
# runtime['batch_limit'] if runtime is not None else 250,
# runtime['query_limit'] if runtime is not None else 100
# )))
job_data = self.driver.batch(
self.args.batch, self.args.wdl, self.args.workflow_inputs,
self.args.workflow_options,
runtime['batch_limit'] if runtime is not None else 250,
runtime['query_limit'] if runtime is not None else 100)
logging.info("Copying execution output file")
with open('workflows.json', 'w') as w:
json.dump(job_data, w, indent=2)
file_util.gsutil_cp(['workflows.json'],
self.args.output_dir + '/')
for data in job_data:
logging.info(
"Workflow %s exited with status %s" %
(data['workflow_id'], data['workflow_status']))
# if data['workflow_output'] is not None:
# metadata_filename = '%s.%s' % (
# data['workflow_id'],
# WDL_RUN_METADATA_FILE
# )
# with open(metadata_filename, 'w') as w:
# json.dump(
# data['workflow_metadata'],
# w,
# indent=2
# )
# file_util.gsutil_cp([metadata_filename], self.args.output_dir+'/')
logging.info("Run complete")
statuses = {wf['workflow_status'] for wf in job_data}
# infer status then get submission object
file_util.gsutil_cp([os.environ['SUBMISSION_DATA_PATH']],
'submission.json')
with open('submission.json') as r:
submission_data = json.load(r)
if len(statuses) == 1 and 'Succeeded' in statuses:
submission_data['status'] = 'Succeeded'
elif 'Aborted' in statuses:
submission_data['status'] = 'Aborted'
elif 'Failed' in statuses:
submission_data['status'] = 'Failed'
else:
submission_data['status'] = 'Error'
submission_data['error-details'] = {
'message':
"Unable to resolve the final status of this submission",
'encountered-statuses': list(statuses)
}
self.driver.log('Unknown job exit status',
json=job_data,
severity='WARNING')
with open('submission.json', 'w') as w:
json.dump(submission_data, w, indent=2)
file_util.gsutil_cp(['submission.json'],
os.environ['SUBMISSION_DATA_PATH'])
self.driver.logger.log('Cromwell complete. Flushing logs',
json=submission_data,
severity='DEBUG')
return
except:
self.driver.logger.log_exception()
logging.error("Batch submission failed: " +
traceback.format_exc())
file_util.gsutil_cp([os.environ['SUBMISSION_DATA_PATH']],
'submission.json')
with open('submission.json') as r:
submission_data = json.load(r)
submission_data['status'] = 'Error'
submission_data['error-details'] = {
'message':
"Lapdog wrappers encountered an unhandled exception",
'stack-trace': traceback.format_exc()
}
with open('submission.json', 'w') as w:
json.dump(submission_data, w, indent=2)
file_util.gsutil_cp(['submission.json'],
os.environ['SUBMISSION_DATA_PATH'])
return
logging.info("starting")
# Submit the job to the local Cromwell server
(result, metadata) = self.driver.submit(self.args.wdl,
self.args.workflow_inputs,
self.args.workflow_options)
logging.info(result)
# Copy run metadata and output files to the output directory
self.copy_workflow_metadata(metadata, WDL_RUN_METADATA_FILE)
logging.info("run complete")