Ejemplo n.º 1
0
    def test_operator_trigger_executes_runs_individually(self, create_jobs_from_chaining):
        for op_run in OperatorRun.objects.prefetch_related("runs").all():
            for t in op_run.operator.from_triggers.all():
                if t.run_type == TriggerRunType.INDIVIDUAL:
                    operator_run = op_run
                    trigger = t
                    break

        run_ids = [run.id for run in operator_run.runs.all()]
        for run_id in run_ids:
            complete_job(run_id, "done")

        calls = [
            call(trigger.to_operator.pk,
                 trigger.from_operator.pk,
                 [run_ids[0]], job_group_id=None,
                 parent=str(operator_run.id)
                 ),
            call(trigger.to_operator.pk,
                 trigger.from_operator.pk,
                 [run_ids[1]], job_group_id=None,
                 parent=str(operator_run.id)
                 ),
            call(trigger.to_operator.pk,
                 trigger.from_operator.pk,
                 [run_ids[2]], job_group_id=None,
                 parent=str(operator_run.id)
                 )
        ]

        create_jobs_from_chaining.delay.assert_has_calls(calls, any_order=True)
        process_triggers()
        operator_run.refresh_from_db()
        self.assertEqual(operator_run.status, RunStatus.COMPLETED)
Ejemplo n.º 2
0
    def test_operator_trigger_does_not_create_next_operator_run_when_too_few_runs_completed(self,
                                                                                            create_jobs_from_chaining):
        operator_run = OperatorRun.objects.prefetch_related("runs").first()
        run_ids = list(operator_run.runs.order_by('id').values_list('id', flat=True))
        complete_job(run_ids.pop(), "done")

        process_triggers()

        create_jobs_from_chaining.delay.assert_not_called()
Ejemplo n.º 3
0
    def test_operator_trigger_does_not_create_next_operator_run_when_too_few_runs_completed(
        self, create_jobs_from_chaining, memcache_task_lock, send_notification
    ):
        memcache_task_lock.return_value = True
        send_notification.return_value = False
        operator_run = OperatorRun.objects.prefetch_related("runs").first()
        run_ids = list(operator_run.runs.order_by("id").values_list("id", flat=True))
        complete_job(run_ids.pop(), "done")

        process_triggers()

        create_jobs_from_chaining.delay.assert_not_called()
Ejemplo n.º 4
0
    def test_operator_trigger_creates_next_operator_run_when_90percent_runs_completed(self, create_jobs_from_chaining):
        operator_run = OperatorRun.objects.prefetch_related("runs").first()
        run_ids = list(operator_run.runs.order_by('id').values_list('id', flat=True))
        for run_id in run_ids:
            complete_job(run_id, "done")

        process_triggers()
        operator_run.refresh_from_db()
        trigger = operator_run.operator.from_triggers.first()

        create_jobs_from_chaining.delay.assert_called_once_with(trigger.to_operator.pk,
                                                                trigger.from_operator.pk,
                                                                run_ids,
                                                                job_group_id=None,
                                                                job_group_notifier_id=None,
                                                                parent=str(operator_run.id))
        self.assertEqual(operator_run.status, RunStatus.COMPLETED)
Ejemplo n.º 5
0
 def test_run_complete_job(
     self, mock_populate_job_group_notifier, mock_get_pipeline, memcache_task_lock, send_notification
 ):
     with open("runner/tests/run/pair-workflow.cwl", "r") as f:
         app = json.load(f)
     with open("runner/tests/run/inputs.json", "r") as f:
         inputs = json.load(f)
     mock_populate_job_group_notifier.return_value = None
     mock_get_pipeline.return_value = app
     memcache_task_lock.return_value = True
     send_notification.return_value = False
     run = RunObjectFactory.from_definition(str(self.run.id), inputs)
     run.to_db()
     operator_run = OperatorRun.objects.first()
     operator_run.runs.add(run.run_obj)
     num_completed_runs = operator_run.num_completed_runs
     complete_job(run.run_id, self.outputs)
     operator_run.refresh_from_db()
     self.assertEqual(operator_run.num_completed_runs, num_completed_runs + 1)
     run_obj = RunObjectFactory.from_db(run.run_id)
     file_obj = File.objects.filter(path=self.outputs["maf"]["location"].replace("file://", "")).first()
     run_obj.to_db()
     for out in run_obj.outputs:
         if out.name == "maf":
             self.assertEqual(out.value["location"], self.outputs["maf"]["location"])
             self.assertEqual(FileProcessor.get_bid_from_file(file_obj), out.db_value["location"])
     port = Port.objects.filter(run_id=run_obj.run_id, name="bams").first()
     self.assertEqual(len(port.files.all()), 4)
     expected_result = (
         "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam",
         "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai",
         "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam",
         "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai",
     )
     self.assertTrue(port.files.all()[0].path in expected_result)
     self.assertTrue(port.files.all()[1].path in expected_result)
     self.assertTrue(port.files.all()[2].path in expected_result)
     self.assertTrue(port.files.all()[3].path in expected_result)
Ejemplo n.º 6
0
 def test_run_complete_job(self, mock_get_pipeline):
     with open('runner/tests/run/pair-workflow.cwl', 'r') as f:
         app = json.load(f)
     with open('runner/tests/run/inputs.json', 'r') as f:
         inputs = json.load(f)
     mock_get_pipeline.return_value = app
     run = RunObject.from_cwl_definition(str(self.run.id), inputs)
     run.to_db()
     operator_run = OperatorRun.objects.first()
     operator_run.runs.add(run.run_obj)
     num_completed_runs = operator_run.num_completed_runs
     complete_job(run.run_id, self.outputs)
     operator_run.refresh_from_db()
     self.assertEqual(operator_run.num_completed_runs,
                      num_completed_runs + 1)
     run_obj = RunObject.from_db(run.run_id)
     file_obj = File.objects.filter(path=self.outputs['maf']['location'].
                                    replace('file://', '')).first()
     run_obj.to_db()
     for out in run_obj.outputs:
         if out.name == 'maf':
             self.assertEqual(out.value['location'],
                              self.outputs['maf']['location'])
             self.assertEqual(FileProcessor.get_bid_from_file(file_obj),
                              out.db_value['location'])
     port = Port.objects.filter(run_id=run_obj.run_id, name='bams').first()
     self.assertEqual(len(port.files.all()), 4)
     expected_result = (
         '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam',
         '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai',
         '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam',
         '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai'
     )
     self.assertTrue(port.files.all()[0].path in expected_result)
     self.assertTrue(port.files.all()[1].path in expected_result)
     self.assertTrue(port.files.all()[2].path in expected_result)
     self.assertTrue(port.files.all()[3].path in expected_result)