def precreate_output_collections(self, history, params):
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
# with the collection and wrap everything up so can evaluate output
# label.
trans = self.trans
params.update(
self.collection_info.collections
) # Replace datasets with source collections for labelling outputs.
collection_instances = {}
implicit_inputs = self.implicit_inputs
implicit_collection_jobs = model.ImplicitCollectionJobs()
for output_name, output in self.tool.outputs.items():
if filter_output(output, self.example_params):
continue
output_collection_name = self.output_name(trans, history, params,
output)
effective_structure = self._mapped_output_structure(trans, output)
collection_instance = trans.app.dataset_collections_service.precreate_dataset_collection_instance(
trans=trans,
parent=history,
name=output_collection_name,
implicit_inputs=implicit_inputs,
implicit_output_name=output_name,
structure=effective_structure,
)
collection_instance.implicit_collection_jobs = implicit_collection_jobs
collection_instances[output_name] = collection_instance
trans.sa_session.add(collection_instance)
# Needed to flush the association created just above with
# job.add_output_dataset_collection.
trans.sa_session.flush()
self.implicit_collections = collection_instances
def precreate_output_collections(self, history, params):
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
# with the collection and wrap everything up so can evaluate output
# label.
trans = self.trans
params.update(
self.collection_info.collections
) # Replace datasets with source collections for labelling outputs.
collection_instances = {}
implicit_inputs = self.implicit_inputs
implicit_collection_jobs = model.ImplicitCollectionJobs()
# trying to guess these filters at the collection levell is tricky because
# the filter condition could vary from element to element. Just do best we
# we can for now.
example_params = self.example_params.copy()
# walk through and optional replace runtime values with None, assume they
# would have been replaced by now if they were going to be set.
def replace_optional_runtime_values(path, key, value):
if is_runtime_value(value):
return key, None
return key, value
example_params = remap(example_params,
visit=replace_optional_runtime_values)
for output_name, output in self.tool.outputs.items():
if filter_output(self.tool, output, example_params):
continue
output_collection_name = self.output_name(trans, history, params,
output)
effective_structure = self._mapped_output_structure(trans, output)
collection_instance = trans.app.dataset_collection_manager.precreate_dataset_collection_instance(
trans=trans,
parent=history,
name=output_collection_name,
structure=effective_structure,
implicit_inputs=implicit_inputs,
implicit_output_name=output_name,
completed_collection=self.completed_jobs,
)
collection_instance.implicit_collection_jobs = implicit_collection_jobs
collection_instances[output_name] = collection_instance
trans.sa_session.add(collection_instance)
# Needed to flush the association created just above with
# job.add_output_dataset_collection.
trans.sa_session.flush()
self.implicit_collections = collection_instances
def test_export_collection_with_mapping_history():
app, sa_session, h = _setup_history_for_export(
"Collection Mapping History")
d1, d2, d3, d4 = _create_datasets(sa_session, h, 4)
c1 = model.DatasetCollection(collection_type="list")
hc1 = model.HistoryDatasetCollectionAssociation(
history=h, hid=1, collection=c1, name="HistoryCollectionTest1")
dce1 = model.DatasetCollectionElement(collection=c1,
element=d1,
element_identifier="el1",
element_index=0)
dce2 = model.DatasetCollectionElement(collection=c1,
element=d2,
element_identifier="el2",
element_index=1)
c2 = model.DatasetCollection(collection_type="list")
hc2 = model.HistoryDatasetCollectionAssociation(
history=h, hid=2, collection=c2, name="HistoryCollectionTest2")
dce3 = model.DatasetCollectionElement(collection=c2,
element=d3,
element_identifier="el1",
element_index=0)
dce4 = model.DatasetCollectionElement(collection=c2,
element=d4,
element_identifier="el2",
element_index=1)
hc2.add_implicit_input_collection("input1", hc1)
j1 = model.Job()
j1.user = h.user
j1.tool_id = "cat1"
j1.add_input_dataset("input1", d1)
j1.add_output_dataset("out_file1", d3)
j2 = model.Job()
j2.user = h.user
j2.tool_id = "cat1"
j2.add_input_dataset("input1", d2)
j2.add_output_dataset("out_file1", d4)
sa_session.add(dce1)
sa_session.add(dce2)
sa_session.add(dce3)
sa_session.add(dce4)
sa_session.add(hc1)
sa_session.add(hc2)
sa_session.add(j1)
sa_session.add(j2)
sa_session.flush()
implicit_collection_jobs = model.ImplicitCollectionJobs()
j1.add_output_dataset_collection("out_file1", hc2) # really?
ija1 = model.ImplicitCollectionJobsJobAssociation()
ija1.order_index = 0
ija1.implicit_collection_jobs = implicit_collection_jobs
ija1.job = j1
j2.add_output_dataset_collection("out_file1", hc2) # really?
ija2 = model.ImplicitCollectionJobsJobAssociation()
ija2.order_index = 1
ija2.implicit_collection_jobs = implicit_collection_jobs
ija2.job = j2
sa_session.add(implicit_collection_jobs)
sa_session.add(ija1)
sa_session.add(ija2)
sa_session.flush()
imported_history = _import_export(app, h)
assert len(imported_history.jobs) == 2
imported_job0 = imported_history.jobs[0]
imported_icj = imported_job0.implicit_collection_jobs_association.implicit_collection_jobs
assert imported_icj
assert len(imported_icj.jobs) == 2, len(imported_icj.jobs)
