def precreate_output_collections(self, history, params): # params is just one sample tool param execution with parallelized # collection replaced with a specific dataset. Need to replace this # with the collection and wrap everything up so can evaluate output # label. trans = self.trans params.update( self.collection_info.collections ) # Replace datasets with source collections for labelling outputs. collection_instances = {} implicit_inputs = self.implicit_inputs implicit_collection_jobs = model.ImplicitCollectionJobs() for output_name, output in self.tool.outputs.items(): if filter_output(output, self.example_params): continue output_collection_name = self.output_name(trans, history, params, output) effective_structure = self._mapped_output_structure(trans, output) collection_instance = trans.app.dataset_collections_service.precreate_dataset_collection_instance( trans=trans, parent=history, name=output_collection_name, implicit_inputs=implicit_inputs, implicit_output_name=output_name, structure=effective_structure, ) collection_instance.implicit_collection_jobs = implicit_collection_jobs collection_instances[output_name] = collection_instance trans.sa_session.add(collection_instance) # Needed to flush the association created just above with # job.add_output_dataset_collection. trans.sa_session.flush() self.implicit_collections = collection_instances
def precreate_output_collections(self, history, params): # params is just one sample tool param execution with parallelized # collection replaced with a specific dataset. Need to replace this # with the collection and wrap everything up so can evaluate output # label. trans = self.trans params.update( self.collection_info.collections ) # Replace datasets with source collections for labelling outputs. collection_instances = {} implicit_inputs = self.implicit_inputs implicit_collection_jobs = model.ImplicitCollectionJobs() # trying to guess these filters at the collection levell is tricky because # the filter condition could vary from element to element. Just do best we # we can for now. example_params = self.example_params.copy() # walk through and optional replace runtime values with None, assume they # would have been replaced by now if they were going to be set. def replace_optional_runtime_values(path, key, value): if is_runtime_value(value): return key, None return key, value example_params = remap(example_params, visit=replace_optional_runtime_values) for output_name, output in self.tool.outputs.items(): if filter_output(self.tool, output, example_params): continue output_collection_name = self.output_name(trans, history, params, output) effective_structure = self._mapped_output_structure(trans, output) collection_instance = trans.app.dataset_collection_manager.precreate_dataset_collection_instance( trans=trans, parent=history, name=output_collection_name, structure=effective_structure, implicit_inputs=implicit_inputs, implicit_output_name=output_name, completed_collection=self.completed_jobs, ) collection_instance.implicit_collection_jobs = implicit_collection_jobs collection_instances[output_name] = collection_instance trans.sa_session.add(collection_instance) # Needed to flush the association created just above with # job.add_output_dataset_collection. trans.sa_session.flush() self.implicit_collections = collection_instances
def test_export_collection_with_mapping_history(): app, sa_session, h = _setup_history_for_export( "Collection Mapping History") d1, d2, d3, d4 = _create_datasets(sa_session, h, 4) c1 = model.DatasetCollection(collection_type="list") hc1 = model.HistoryDatasetCollectionAssociation( history=h, hid=1, collection=c1, name="HistoryCollectionTest1") dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="el1", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="el2", element_index=1) c2 = model.DatasetCollection(collection_type="list") hc2 = model.HistoryDatasetCollectionAssociation( history=h, hid=2, collection=c2, name="HistoryCollectionTest2") dce3 = model.DatasetCollectionElement(collection=c2, element=d3, element_identifier="el1", element_index=0) dce4 = model.DatasetCollectionElement(collection=c2, element=d4, element_identifier="el2", element_index=1) hc2.add_implicit_input_collection("input1", hc1) j1 = model.Job() j1.user = h.user j1.tool_id = "cat1" j1.add_input_dataset("input1", d1) j1.add_output_dataset("out_file1", d3) j2 = model.Job() j2.user = h.user j2.tool_id = "cat1" j2.add_input_dataset("input1", d2) j2.add_output_dataset("out_file1", d4) sa_session.add(dce1) sa_session.add(dce2) sa_session.add(dce3) sa_session.add(dce4) sa_session.add(hc1) sa_session.add(hc2) sa_session.add(j1) sa_session.add(j2) sa_session.flush() implicit_collection_jobs = model.ImplicitCollectionJobs() j1.add_output_dataset_collection("out_file1", hc2) # really? ija1 = model.ImplicitCollectionJobsJobAssociation() ija1.order_index = 0 ija1.implicit_collection_jobs = implicit_collection_jobs ija1.job = j1 j2.add_output_dataset_collection("out_file1", hc2) # really? ija2 = model.ImplicitCollectionJobsJobAssociation() ija2.order_index = 1 ija2.implicit_collection_jobs = implicit_collection_jobs ija2.job = j2 sa_session.add(implicit_collection_jobs) sa_session.add(ija1) sa_session.add(ija2) sa_session.flush() imported_history = _import_export(app, h) assert len(imported_history.jobs) == 2 imported_job0 = imported_history.jobs[0] imported_icj = imported_job0.implicit_collection_jobs_association.implicit_collection_jobs assert imported_icj assert len(imported_icj.jobs) == 2, len(imported_icj.jobs)