def create_run_task(run_id, inputs, output_directory=None):
logger.info("Creating and validating Run for %s" % run_id)
run = RunObject.from_cwl_definition(run_id, inputs)
run.ready()
RunObject.to_db(run)
submit_job.delay(run_id, output_directory)
logger.info("Run %s Ready" % run_id)
def submit_job(run_id, output_directory=None):
resume = None
try:
run = Run.objects.get(id=run_id)
except Run.DoesNotExist:
raise Exception("Failed to submit a run")
if run.resume:
run1 = RunObject.from_db(run_id)
run2 = RunObject.from_db(run.resume)
if run1.equal(run2):
logger.info("Resuming run: %s with execution id: %s" %
(str(run.resume), str(run2.run_obj.execution_id)))
resume = str(run2.run_obj.execution_id)
else:
logger.info("Failed to resume runs not equal")
app = {
"github": {
"repository": run.app.github,
"entrypoint": run.app.entrypoint,
"version": run.app.version
}
}
inputs = dict()
for port in run.port_set.filter(port_type=PortType.INPUT).all():
inputs[port.name] = port.value
if not output_directory:
output_directory = os.path.join(run.app.output_directory, str(run_id))
logger.info("Job %s ready for submitting" % run_id)
if resume:
url = urljoin(settings.RIDGEBACK_URL,
'/v0/jobs/{id}/resume/'.format(id=resume))
job = {'root_dir': output_directory}
response = requests.post(url, json=job)
else:
url = settings.RIDGEBACK_URL + '/v0/jobs/'
job = {'app': app, 'inputs': inputs, 'root_dir': output_directory}
response = requests.post(url, json=job)
if response.status_code == 201:
run.execution_id = response.json()['id']
run.status = RunStatus.RUNNING
logger.info("Job %s successfully submitted with id:%s" %
(run_id, run.execution_id))
run.save()
else:
logger.info("Failed to submit job %s" % run_id)
run.status = RunStatus.FAILED
run.save()
def test_run_creation_from_cwl(self, mock_get_pipeline):
with open('runner/tests/run/pair-workflow.cwl', 'r') as f:
app = json.load(f)
with open('runner/tests/run/inputs.json', 'r') as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObject.from_cwl_definition(str(self.run.id), inputs)
run.ready()
for inp in run.inputs:
if inp.name == 'pair':
self.assertEqual(inp.db_value[0]['R1'][0]['location'],
'bid://%s' % str(self.file1.id))
self.assertEqual(inp.value[0]['R1'][0]['path'],
self.file1.path)
self.assertEqual(inp.db_value[0]['R2'][0]['location'],
'bid://%s' % str(self.file2.id))
self.assertEqual(inp.value[0]['R2'][0]['path'],
self.file2.path)
self.assertEqual(inp.db_value[1]['R1'][0]['location'],
'bid://%s' % str(self.file3.id))
self.assertEqual(inp.value[1]['R1'][0]['path'],
self.file3.path)
self.assertEqual(inp.db_value[1]['R2'][0]['location'],
'bid://%s' % str(self.file4.id))
self.assertEqual(inp.value[1]['R2'][0]['path'],
self.file4.path)
def test_run_fail_job(self, mock_get_pipeline):
with open('runner/tests/run/pair-workflow.cwl', 'r') as f:
app = json.load(f)
with open('runner/tests/run/inputs.json', 'r') as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObject.from_cwl_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_failed_runs = operator_run.num_failed_runs
fail_job(run.run_id, {'details': 'Error has happened'})
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_failed_runs, num_failed_runs + 1)
run_obj = RunObject.from_db(run.run_id)
self.assertEqual(run_obj.message, {'details': 'Error has happened'})
def fail_job(run_id, error_message):
run = RunObject.from_db(run_id)
run.fail(error_message)
run.to_db()
job_group_notifier = run.job_group_notifier
job_group_notifier_id = str(
job_group_notifier.id) if job_group_notifier else None
ci_review = SetCIReviewEvent(job_group_notifier_id).to_dict()
send_notification.delay(ci_review)
_job_finished_notify(run)
def test_run_to_db(self, mock_get_pipeline):
with open('runner/tests/run/pair-workflow.cwl', 'r') as f:
app = json.load(f)
with open('runner/tests/run/inputs.json', 'r') as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObject.from_cwl_definition(str(self.run.id), inputs)
run.to_db()
try:
run_obj = Run.objects.get(id=run.run_id)
except Run.DoesNotExist as e:
pass
self.assertEqual(str(run_obj.id), run.run_id)
def test_run_complete_job(self, mock_get_pipeline):
with open('runner/tests/run/pair-workflow.cwl', 'r') as f:
app = json.load(f)
with open('runner/tests/run/inputs.json', 'r') as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObject.from_cwl_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_completed_runs = operator_run.num_completed_runs
complete_job(run.run_id, self.outputs)
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_completed_runs,
num_completed_runs + 1)
run_obj = RunObject.from_db(run.run_id)
file_obj = File.objects.filter(path=self.outputs['maf']['location'].
replace('file://', '')).first()
run_obj.to_db()
for out in run_obj.outputs:
if out.name == 'maf':
self.assertEqual(out.value['location'],
self.outputs['maf']['location'])
self.assertEqual(FileProcessor.get_bid_from_file(file_obj),
out.db_value['location'])
port = Port.objects.filter(run_id=run_obj.run_id, name='bams').first()
self.assertEqual(len(port.files.all()), 4)
expected_result = (
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam',
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai',
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam',
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai'
)
self.assertTrue(port.files.all()[0].path in expected_result)
self.assertTrue(port.files.all()[1].path in expected_result)
self.assertTrue(port.files.all()[2].path in expected_result)
self.assertTrue(port.files.all()[3].path in expected_result)
def populate_run_samples(apps, _):
Run = apps.get_model('runner', 'Run')
for run in Run.objects.all():
samples = set()
try:
run_obj = RunObject.from_db(run.id)
except Exception as e:
print("Run %s can't be migrated" % str(run.id))
for p in run_obj.inputs:
for f in p.files:
file_obj = FileProcessor.get_file_obj(f)
if file_obj.sample:
samples.add(file_obj.sample)
run_obj.samples = list(samples)
run_obj.to_db()
def complete_job(run_id, outputs):
run = RunObject.from_db(run_id)
run.complete(outputs)
run.to_db()
job_group = run.job_group
job_group_id = str(job_group.id) if job_group else None
_job_finished_notify(run)
for trigger in run.run_obj.operator_run.operator.from_triggers.filter(
run_type=TriggerRunType.INDIVIDUAL):
create_jobs_from_chaining.delay(trigger.to_operator_id,
trigger.from_operator_id, [run_id],
job_group_id=job_group_id)
def create_run_task(run_id, inputs, output_directory=None):
logger.info("Creating and validating Run")
try:
run = RunObject.from_cwl_definition(run_id, inputs)
run.ready()
except RunCreateException as e:
run = RunObject.from_db(run_id)
run.fail({'details': str(e)})
RunObject.to_db(run)
logger.info("Run %s Failed" % run_id)
else:
RunObject.to_db(run)
submit_job.delay(run_id, output_directory)
logger.info("Run %s Ready" % run_id)
def post(self, request):
run_id = request.data.get('run')
run = RunObject.from_db(run_id)
inputs = dict()
for port in run.inputs:
inputs[port.name] = port.db_value
data = dict(app=str(run.run_obj.app.id),
inputs=inputs,
tags=run.tags,
job_group_id=run.job_group.id,
job_group_notifier_id=run.job_group_notifier.id,
resume=run_id)
serializer = APIRunCreateSerializer(data=data,
context={'request': request})
if serializer.is_valid():
new_run = serializer.save()
response = RunSerializerFull(new_run)
create_run_task.delay(response.data['id'], data['inputs'])
job_group_notifier_id = str(new_run.job_group_notifier_id)
self._send_notifications(job_group_notifier_id, new_run)
return Response(response.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def complete_job(run_id, outputs):
run = RunObject.from_db(run_id)
if run.run_obj.is_completed:
return
try:
run.complete(outputs)
except Exception as e:
fail_job(run_id, str(e))
return
run.to_db()
job_group = run.job_group
job_group_id = str(job_group.id) if job_group else None
_job_finished_notify(run)
for trigger in run.run_obj.operator_run.operator.from_triggers.filter(
run_type=TriggerRunType.INDIVIDUAL):
create_jobs_from_chaining.delay(trigger.to_operator_id,
trigger.from_operator_id, [run_id],
job_group_id=job_group_id,
parent=str(run.run_obj.operator_run.id)
if run.run_obj.operator_run else None)