def addQuantum(self, quantum: Quantum):
config = self.taskDef.config
connectionClass = config.connections.ConnectionsClass
connectionInstance = connectionClass(config=config)
# This will raise if one of the check conditions is not met, which is the intended
# behavior
result = connectionInstance.adjustQuantum(quantum.predictedInputs)
quantum._predictedInputs = NamedKeyDict(result)
# If this function has reached this far add the quantum
self.quanta.append(quantum)
def testConstructor(self):
"""Test of constructor.
"""
# Quantum specific arguments
run = None # TODO add Run
taskName = "some.task.object" # can't use a real PipelineTask due to inverted package dependency
# Base class arguments
startTime = datetime(2018, 1, 1)
endTime = datetime(2018, 1, 2)
host = "localhost"
quantum = Quantum(taskName=taskName, run=run, startTime=startTime, endTime=endTime, host=host)
self.assertEqual(quantum.taskName, taskName)
self.assertEqual(quantum.run, run)
self.assertEqual(quantum.predictedInputs, NamedKeyDict())
self.assertEqual(quantum.actualInputs, NamedKeyDict())
self.assertIsNone(quantum.dataId)
self.assertIsNone(quantum.id)
self.assertEqual(quantum.startTime, startTime)
self.assertEqual(quantum.endTime, endTime)
self.assertEqual(quantum.host, host)
def unpackRefs(self) -> NamedKeyDict:
"""Unpack nested single-element `DatasetRef` lists into a new
dictionary.
This method assumes that each `_DatasetScaffolding.refs` list contains
exactly one `DatasetRef`, as is the case for all "init" datasets.
Returns
-------
dictionary : `NamedKeyDict`
Dictionary mapping `DatasetType` to `DatasetRef`, with both
`DatasetType` instances and string names usable as keys.
"""
return NamedKeyDict((datasetType, scaffolding.refs[0]) for datasetType, scaffolding in self.items())
def fillQuanta(self, registry, inputCollections, *, skipExisting=True):
"""Define quanta for each task by splitting up the datasets associated
with each task data ID.
This method populates `_TaskScaffolding.quanta`.
Parameters
----------
registry : `lsst.daf.butler.Registry`
Registry for the data repository; used for all data ID queries.
inputCollections : `~collections.abc.Mapping`
Mapping from dataset type name to an ordered sequence of
collections to search for that dataset. A `defaultdict` is
recommended for the case where the same collections should be
used for most datasets.
skipExisting : `bool`, optional
If `True` (default), a Quantum is not created if all its outputs
already exist.
"""
for task in self.tasks:
for quantumDataId in task.dataIds:
# Identify the (regular) inputs that correspond to the Quantum
# with this data ID. These are those whose data IDs have the
# same values for all dimensions they have in common.
# We do this data IDs expanded to include implied dimensions,
# which is why _DatasetScaffolding.dimensions is thus expanded
# even though DatasetType.dimensions is not.
inputs = NamedKeyDict()
for datasetType, scaffolding in task.inputs.items():
inputs[datasetType] = [ref for ref, dataId in zip(scaffolding.refs, scaffolding.dataIds)
if quantumDataId.matches(dataId)]
# Same for outputs.
outputs = NamedKeyDict()
allOutputsPresent = True
for datasetType, scaffolding in task.outputs.items():
outputs[datasetType] = []
for ref, dataId in zip(scaffolding.refs, scaffolding.dataIds):
if quantumDataId.matches(dataId):
if ref.id is None:
allOutputsPresent = False
else:
assert skipExisting, "Existing outputs should have already been identified."
if not allOutputsPresent:
raise OutputExistsError(f"Output {datasetType.name} with data ID "
f"{dataId} already exists, but other outputs "
f"for task with label {task.taskDef.label} "
f"and data ID {quantumDataId} do not.")
outputs[datasetType].append(ref)
if allOutputsPresent and skipExisting:
continue
# Look up prerequisite datasets in the input collection(s).
# These may have dimensions that extend beyond those we queried
# for originally, because we want to permit those data ID
# values to differ across quanta and dataset types.
# For example, the same quantum may have a flat and bias with
# a different calibration_label, or a refcat with a skypix
# value that overlaps the quantum's data ID's region, but not
# the user expression used for the initial query.
for datasetType, scaffolding in task.prerequisites.items():
refs = list(
registry.queryDatasets(
datasetType,
collections=inputCollections[datasetType.name],
dataId=quantumDataId,
deduplicate=True,
expand=True,
)
)
inputs[datasetType] = refs
task.addQuantum(
Quantum(
taskName=task.taskDef.taskName,
taskClass=task.taskDef.taskClass,
dataId=quantumDataId,
initInputs=task.initInputs.unpackRefs(),
predictedInputs=inputs,
outputs=outputs,
)
)
def __init__(self, iterable=None):
list.__init__(self, iterable or [])
self.initInputs = NamedKeyDict()
self.initIntermediates = NamedKeyDict()
self.initOutputs = NamedKeyDict()