def _register_file(val, size, group_id, metadata, file_list): file_obj = copy.deepcopy(val) file_obj.pop("basename", None) file_obj.pop("nameroot", None) file_obj.pop("nameext", None) uri = file_obj.pop('location', None) checksum = file_obj.pop("checksum", None) try: file_obj_db = FileProcessor.create_file_obj( uri, size, checksum, group_id, metadata) except FileConflictException as e: logger.warning(str(e)) # TODO: Check what to do in case file already exist in DB. file_obj_db = FileProcessor.get_file_obj(uri) FileProcessor.update_file(file_obj_db, file_obj_db.path, metadata) secondary_files = file_obj.pop('secondaryFiles', []) secondary_file_list = [] secondary_files_obj = PortProcessor.process_files( secondary_files, PortAction.REGISTER_OUTPUT_FILES, group_id=group_id, metadata=metadata, file_list=secondary_file_list) if secondary_files_obj: file_obj['secondaryFiles'] = secondary_files_obj file_obj['location'] = FileProcessor.get_bid_from_file(file_obj_db) if file_list is not None: file_list.append('bid://%s' % FileProcessor.get_bid_from_file(file_obj_db)) file_list.extend([f['location'] for f in secondary_files_obj]) return file_obj
def from_db(cls, port_id): try: port = Port.objects.get(id=port_id) except Port.DoesNotExist: raise PortObjectConstructException('Port with id:') return cls(str(port.run.id), port.name, port.port_type, port.schema, port.secondary_files, port.db_value, port.value, [FileProcessor.get_bid_from_file(f) for f in port.files.all()], port_id=port_id, notify=port.notify)
def test_run_complete_job( self, mock_populate_job_group_notifier, mock_get_pipeline, memcache_task_lock, send_notification ): with open("runner/tests/run/pair-workflow.cwl", "r") as f: app = json.load(f) with open("runner/tests/run/inputs.json", "r") as f: inputs = json.load(f) mock_populate_job_group_notifier.return_value = None mock_get_pipeline.return_value = app memcache_task_lock.return_value = True send_notification.return_value = False run = RunObjectFactory.from_definition(str(self.run.id), inputs) run.to_db() operator_run = OperatorRun.objects.first() operator_run.runs.add(run.run_obj) num_completed_runs = operator_run.num_completed_runs complete_job(run.run_id, self.outputs) operator_run.refresh_from_db() self.assertEqual(operator_run.num_completed_runs, num_completed_runs + 1) run_obj = RunObjectFactory.from_db(run.run_id) file_obj = File.objects.filter(path=self.outputs["maf"]["location"].replace("file://", "")).first() run_obj.to_db() for out in run_obj.outputs: if out.name == "maf": self.assertEqual(out.value["location"], self.outputs["maf"]["location"]) self.assertEqual(FileProcessor.get_bid_from_file(file_obj), out.db_value["location"]) port = Port.objects.filter(run_id=run_obj.run_id, name="bams").first() self.assertEqual(len(port.files.all()), 4) expected_result = ( "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam", "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai", "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam", "/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai", ) self.assertTrue(port.files.all()[0].path in expected_result) self.assertTrue(port.files.all()[1].path in expected_result) self.assertTrue(port.files.all()[2].path in expected_result) self.assertTrue(port.files.all()[3].path in expected_result)
def test_run_complete_job(self, mock_get_pipeline): with open('runner/tests/run/pair-workflow.cwl', 'r') as f: app = json.load(f) with open('runner/tests/run/inputs.json', 'r') as f: inputs = json.load(f) mock_get_pipeline.return_value = app run = RunObject.from_cwl_definition(str(self.run.id), inputs) run.to_db() operator_run = OperatorRun.objects.first() operator_run.runs.add(run.run_obj) num_completed_runs = operator_run.num_completed_runs complete_job(run.run_id, self.outputs) operator_run.refresh_from_db() self.assertEqual(operator_run.num_completed_runs, num_completed_runs + 1) run_obj = RunObject.from_db(run.run_id) file_obj = File.objects.filter(path=self.outputs['maf']['location']. replace('file://', '')).first() run_obj.to_db() for out in run_obj.outputs: if out.name == 'maf': self.assertEqual(out.value['location'], self.outputs['maf']['location']) self.assertEqual(FileProcessor.get_bid_from_file(file_obj), out.db_value['location']) port = Port.objects.filter(run_id=run_obj.run_id, name='bams').first() self.assertEqual(len(port.files.all()), 4) expected_result = ( '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bam', '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_1.rg.md.abra.printreads.bai', '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bam', '/output/argos_pair_workflow/425194f6-a974-4c2f-995f-f27d7ba54ddc/outputs/test_2.rg.md.abra.printreads.bai' ) self.assertTrue(port.files.all()[0].path in expected_result) self.assertTrue(port.files.all()[1].path in expected_result) self.assertTrue(port.files.all()[2].path in expected_result) self.assertTrue(port.files.all()[3].path in expected_result)