def on_text(self):
if self._on_text is None:
collection_names = [
"collection %d" % c.hid
for c in self.collection_info.collections.values()
]
self._on_text = on_text_for_names(collection_names)
return self._on_text
def create_output_collections( self, trans, history, params ):
# TODO: Move this function - it doesn't belong here but it does need
# the information in this class and potential extensions.
if self.failed_jobs > 0:
return []
structure = self.collection_info.structure
collections = self.collection_info.collections.values()
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
# with the collection and wrap everything up so can evaluate output
# label.
params.update( self.collection_info.collections ) # Replace datasets
# with source collections
# for labelling outputs.
collection_names = map( lambda c: "collection %d" % c.hid, collections )
on_text = on_text_for_names( collection_names )
collections = {}
implicit_inputs = list(self.collection_info.collections.iteritems())
for output_name, outputs_datasets in self.output_datasets_by_output_name.iteritems():
if not len( structure ) == len( outputs_datasets ):
# Output does not have the same structure, if all jobs were
# successfully submitted this shouldn't have happened.
log.warn( "Problem matching up datasets while attempting to create implicit dataset collections")
continue
output = self.tool.outputs[ output_name ]
element_identifiers_for_datasets = structure.element_identifiers_for_datasets( trans, outputs_datasets )
implicit_collection_info = dict(
implicit_inputs=implicit_inputs,
implicit_output_name=output_name,
outputs_datasets=outputs_datasets
)
try:
output_collection_name = self.tool_action.get_output_name(
output,
dataset=None,
tool=self.tool,
on_text=on_text,
trans=trans,
params=params,
incoming=None,
job_params=None,
)
except Exception:
output_collection_name = "%s across %s" % ( self.tool.name, on_text )
collection = trans.app.dataset_collections_service.create(
trans=trans,
parent=history,
name=output_collection_name,
element_identifiers=element_identifiers_for_datasets[ "element_identifiers" ],
collection_type=structure.collection_type_description.collection_type,
implicit_collection_info=implicit_collection_info,
)
collections[ output_name ] = collection
self.created_collections = collections
def create_output_collections(self, trans, history, params):
# TODO: Move this function - it doesn't belong here but it does need
# the information in this class and potential extensions.
if self.failed_jobs > 0:
return []
structure = self.collection_info.structure
collections = self.collection_info.collections.values()
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
# with the collection and wrap everything up so can evaluate output
# label.
params.update(
self.collection_info.collections
) # Replace datasets with source collections for labelling outputs.
collection_names = map(lambda c: "collection %d" % c.hid, collections)
on_text = on_text_for_names(collection_names)
collections = {}
implicit_inputs = list(self.collection_info.collections.iteritems())
for output_name, outputs in self.outputs_by_output_name.iteritems():
if not len(structure) == len(outputs):
# Output does not have the same structure, if all jobs were
# successfully submitted this shouldn't have happened.
log.warning("Problem matching up datasets while attempting to create implicit dataset collections")
continue
output = self.tool.outputs[output_name]
element_identifiers = structure.element_identifiers_for_outputs(trans, outputs)
implicit_collection_info = dict(
implicit_inputs=implicit_inputs, implicit_output_name=output_name, outputs=outputs
)
try:
output_collection_name = self.tool.tool_action.get_output_name(
output,
dataset=None,
tool=self.tool,
on_text=on_text,
trans=trans,
history=history,
params=params,
incoming=None,
job_params=None,
)
except Exception:
output_collection_name = "%s across %s" % (self.tool.name, on_text)
child_element_identifiers = element_identifiers["element_identifiers"]
collection_type = element_identifiers["collection_type"]
collection = trans.app.dataset_collections_service.create(
trans=trans,
parent=history,
name=output_collection_name,
element_identifiers=child_element_identifiers,
collection_type=collection_type,
implicit_collection_info=implicit_collection_info,
)
for job in self.successful_jobs:
# TODO: Think through this, may only want this for output
# collections - or we may be already recording data in some
# other way.
if job not in trans.sa_session:
job = trans.sa_session.query(trans.app.model.Job).get(job.id)
job.add_output_dataset_collection(output_name, collection)
collections[output_name] = collection
# Needed to flush the association created just above with
# job.add_output_dataset_collection.
trans.sa_session.flush()
self.implicit_collections = collections
def create_output_collections(self, trans, history, params):
# TODO: Move this function - it doesn't belong here but it does need
# the information in this class and potential extensions.
if self.failed_jobs > 0:
return []
structure = self.collection_info.structure
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
# with the collection and wrap everything up so can evaluate output
# label.
params.update(
self.collection_info.collections
) # Replace datasets with source collections for labelling outputs.
collection_names = [
"collection %d" % c.hid
for c in self.collection_info.collections.values()
]
on_text = on_text_for_names(collection_names)
collections = {}
implicit_inputs = list(self.collection_info.collections.items())
for output_name, outputs in self.outputs_by_output_name.items():
if not len(structure) == len(outputs):
# Output does not have the same structure, if all jobs were
# successfully submitted this shouldn't have happened.
log.warning(
"Problem matching up datasets while attempting to create implicit dataset collections"
)
continue
output = self.tool.outputs[output_name]
element_identifiers = structure.element_identifiers_for_outputs(
trans, outputs)
implicit_collection_info = dict(implicit_inputs=implicit_inputs,
implicit_output_name=output_name,
outputs=outputs)
try:
output_collection_name = self.tool.tool_action.get_output_name(
output,
dataset=None,
tool=self.tool,
on_text=on_text,
trans=trans,
history=history,
params=params,
incoming=None,
job_params=None,
)
except Exception:
output_collection_name = "%s across %s" % (self.tool.name,
on_text)
child_element_identifiers = element_identifiers[
"element_identifiers"]
collection_type = element_identifiers["collection_type"]
collection = trans.app.dataset_collections_service.create(
trans=trans,
parent=history,
name=output_collection_name,
element_identifiers=child_element_identifiers,
collection_type=collection_type,
implicit_collection_info=implicit_collection_info,
)
for job in self.successful_jobs:
# TODO: Think through this, may only want this for output
# collections - or we may be already recording data in some
# other way.
if job not in trans.sa_session:
job = trans.sa_session.query(trans.app.model.Job).get(
job.id)
job.add_output_dataset_collection(output_name, collection)
collections[output_name] = collection
# Needed to flush the association created just above with
# job.add_output_dataset_collection.
trans.sa_session.flush()
self.implicit_collections = collections
def assert_on_text_is(expected, *names):
on_text = on_text_for_names(names)
assert on_text == expected, "Wrong on text value %s, expected %s" % (
on_text, expected)
def assert_on_text_is( expected, *names ):
on_text = on_text_for_names( names )
assert on_text == expected, "Wrong on text value %s, expected %s" % ( on_text, expected )
def on_text(self):
if self._on_text is None:
collection_names = ["collection %d" % c.hid for c in self.collection_info.collections.values()]
self._on_text = on_text_for_names(collection_names)
return self._on_text
def assert_on_text_is(expected, *names):
on_text = on_text_for_names(names)
assert on_text == expected, f"Wrong on text value {on_text}, expected {expected}"
def create_output_collections(self, trans, history, params):
# TODO: Move this function - it doesn't belong here but it does need
# the information in this class and potential extensions.
if self.failed_jobs > 0:
return []
structure = self.collection_info.structure
collections = self.collection_info.collections.values()
# params is just one sample tool param execution with parallelized
# collection replaced with a specific dataset. Need to replace this
# with the collection and wrap everything up so can evaluate output
# label.
params.update(self.collection_info.collections) # Replace datasets
# with source collections
# for labelling outputs.
collection_names = map(lambda c: "collection %d" % c.hid, collections)
on_text = on_text_for_names(collection_names)
collections = {}
implicit_inputs = list(self.collection_info.collections.iteritems())
for output_name, outputs_datasets in self.output_datasets_by_output_name.iteritems(
):
if not len(structure) == len(outputs_datasets):
# Output does not have the same structure, if all jobs were
# successfully submitted this shouldn't have happened.
log.warn(
"Problem matching up datasets while attempting to create implicit dataset collections"
)
continue
output = self.tool.outputs[output_name]
element_identifiers_for_datasets = structure.element_identifiers_for_datasets(
trans, outputs_datasets)
implicit_collection_info = dict(implicit_inputs=implicit_inputs,
implicit_output_name=output_name,
outputs_datasets=outputs_datasets)
try:
output_collection_name = self.tool_action.get_output_name(
output,
dataset=None,
tool=self.tool,
on_text=on_text,
trans=trans,
params=params,
incoming=None,
job_params=None,
)
except Exception:
output_collection_name = "%s across %s" % (self.tool.name,
on_text)
collection = trans.app.dataset_collections_service.create(
trans=trans,
parent=history,
name=output_collection_name,
element_identifiers=element_identifiers_for_datasets[
"element_identifiers"],
collection_type=structure.collection_type_description.
collection_type,
implicit_collection_info=implicit_collection_info,
)
collections[output_name] = collection
self.created_collections = collections