def test_operator_trigger_executes_runs_individually(self, create_jobs_from_chaining):
for op_run in OperatorRun.objects.prefetch_related("runs").all():
for t in op_run.operator.from_triggers.all():
if t.run_type == TriggerRunType.INDIVIDUAL:
operator_run = op_run
trigger = t
break
run_ids = [run.id for run in operator_run.runs.all()]
for run_id in run_ids:
complete_job(run_id, "done")
calls = [
call(trigger.to_operator.pk,
trigger.from_operator.pk,
[run_ids[0]], job_group_id=None,
parent=str(operator_run.id)
),
call(trigger.to_operator.pk,
trigger.from_operator.pk,
[run_ids[1]], job_group_id=None,
parent=str(operator_run.id)
),
call(trigger.to_operator.pk,
trigger.from_operator.pk,
[run_ids[2]], job_group_id=None,
parent=str(operator_run.id)
)
]
create_jobs_from_chaining.delay.assert_has_calls(calls, any_order=True)
process_triggers()
operator_run.refresh_from_db()
self.assertEqual(operator_run.status, RunStatus.COMPLETED)
def test_operator_trigger_does_not_create_next_operator_run_when_too_few_runs_completed(self,
create_jobs_from_chaining):
operator_run = OperatorRun.objects.prefetch_related("runs").first()
run_ids = list(operator_run.runs.order_by('id').values_list('id', flat=True))
complete_job(run_ids.pop(), "done")
process_triggers()
create_jobs_from_chaining.delay.assert_not_called()
def test_operator_trigger_does_not_create_next_operator_run_when_too_few_runs_completed(
self, create_jobs_from_chaining, memcache_task_lock, send_notification
):
memcache_task_lock.return_value = True
send_notification.return_value = False
operator_run = OperatorRun.objects.prefetch_related("runs").first()
run_ids = list(operator_run.runs.order_by("id").values_list("id", flat=True))
complete_job(run_ids.pop(), "done")
process_triggers()
create_jobs_from_chaining.delay.assert_not_called()
def test_operator_trigger_creates_next_operator_run_when_90percent_runs_completed(self, create_jobs_from_chaining):
operator_run = OperatorRun.objects.prefetch_related("runs").first()
run_ids = list(operator_run.runs.order_by('id').values_list('id', flat=True))
for run_id in run_ids:
complete_job(run_id, "done")
process_triggers()
operator_run.refresh_from_db()
trigger = operator_run.operator.from_triggers.first()
create_jobs_from_chaining.delay.assert_called_once_with(trigger.to_operator.pk,
trigger.from_operator.pk,
run_ids,
job_group_id=None,
job_group_notifier_id=None,
parent=str(operator_run.id))
self.assertEqual(operator_run.status, RunStatus.COMPLETED)
def test_run_complete_job(
self, mock_populate_job_group_notifier, mock_get_pipeline, memcache_task_lock, send_notification
):
with open("runner/tests/run/pair-workflow.cwl", "r") as f:
app = json.load(f)
with open("runner/tests/run/inputs.json", "r") as f:
inputs = json.load(f)
mock_populate_job_group_notifier.return_value = None
mock_get_pipeline.return_value = app
memcache_task_lock.return_value = True
send_notification.return_value = False
run = RunObjectFactory.from_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_completed_runs = operator_run.num_completed_runs
complete_job(run.run_id, self.outputs)
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_completed_runs, num_completed_runs + 1)
run_obj = RunObjectFactory.from_db(run.run_id)
file_obj = File.objects.filter(path=self.outputs["maf"]["location"].replace("file://", "")).first()
run_obj.to_db()
for out in run_obj.outputs:
if out.name == "maf":
self.assertEqual(out.value["location"], self.outputs["maf"]["location"])
self.assertEqual(FileProcessor.get_bid_from_file(file_obj), out.db_value["location"])
port = Port.objects.filter(run_id=run_obj.run_id, name="bams").first()
self.assertEqual(len(port.files.all()), 4)
expected_result = (
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam",
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai",
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam",
"/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai",
)
self.assertTrue(port.files.all()[0].path in expected_result)
self.assertTrue(port.files.all()[1].path in expected_result)
self.assertTrue(port.files.all()[2].path in expected_result)
self.assertTrue(port.files.all()[3].path in expected_result)
def test_run_complete_job(self, mock_get_pipeline):
with open('runner/tests/run/pair-workflow.cwl', 'r') as f:
app = json.load(f)
with open('runner/tests/run/inputs.json', 'r') as f:
inputs = json.load(f)
mock_get_pipeline.return_value = app
run = RunObject.from_cwl_definition(str(self.run.id), inputs)
run.to_db()
operator_run = OperatorRun.objects.first()
operator_run.runs.add(run.run_obj)
num_completed_runs = operator_run.num_completed_runs
complete_job(run.run_id, self.outputs)
operator_run.refresh_from_db()
self.assertEqual(operator_run.num_completed_runs,
num_completed_runs + 1)
run_obj = RunObject.from_db(run.run_id)
file_obj = File.objects.filter(path=self.outputs['maf']['location'].
replace('file://', '')).first()
run_obj.to_db()
for out in run_obj.outputs:
if out.name == 'maf':
self.assertEqual(out.value['location'],
self.outputs['maf']['location'])
self.assertEqual(FileProcessor.get_bid_from_file(file_obj),
out.db_value['location'])
port = Port.objects.filter(run_id=run_obj.run_id, name='bams').first()
self.assertEqual(len(port.files.all()), 4)
expected_result = (
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam',
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai',
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam',
'/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai'
)
self.assertTrue(port.files.all()[0].path in expected_result)
self.assertTrue(port.files.all()[1].path in expected_result)
self.assertTrue(port.files.all()[2].path in expected_result)
self.assertTrue(port.files.all()[3].path in expected_result)