def addQuantum(self, quantum: Quantum): config = self.taskDef.config connectionClass = config.connections.ConnectionsClass connectionInstance = connectionClass(config=config) # This will raise if one of the check conditions is not met, which is the intended # behavior result = connectionInstance.adjustQuantum(quantum.predictedInputs) quantum._predictedInputs = NamedKeyDict(result) # If this function has reached this far add the quantum self.quanta.append(quantum)
def testConstructor(self): """Test of constructor. """ # Quantum specific arguments run = None # TODO add Run taskName = "some.task.object" # can't use a real PipelineTask due to inverted package dependency # Base class arguments startTime = datetime(2018, 1, 1) endTime = datetime(2018, 1, 2) host = "localhost" quantum = Quantum(taskName=taskName, run=run, startTime=startTime, endTime=endTime, host=host) self.assertEqual(quantum.taskName, taskName) self.assertEqual(quantum.run, run) self.assertEqual(quantum.predictedInputs, NamedKeyDict()) self.assertEqual(quantum.actualInputs, NamedKeyDict()) self.assertIsNone(quantum.dataId) self.assertIsNone(quantum.id) self.assertEqual(quantum.startTime, startTime) self.assertEqual(quantum.endTime, endTime) self.assertEqual(quantum.host, host)
def unpackRefs(self) -> NamedKeyDict: """Unpack nested single-element `DatasetRef` lists into a new dictionary. This method assumes that each `_DatasetScaffolding.refs` list contains exactly one `DatasetRef`, as is the case for all "init" datasets. Returns ------- dictionary : `NamedKeyDict` Dictionary mapping `DatasetType` to `DatasetRef`, with both `DatasetType` instances and string names usable as keys. """ return NamedKeyDict((datasetType, scaffolding.refs[0]) for datasetType, scaffolding in self.items())
def fillQuanta(self, registry, inputCollections, *, skipExisting=True): """Define quanta for each task by splitting up the datasets associated with each task data ID. This method populates `_TaskScaffolding.quanta`. Parameters ---------- registry : `lsst.daf.butler.Registry` Registry for the data repository; used for all data ID queries. inputCollections : `~collections.abc.Mapping` Mapping from dataset type name to an ordered sequence of collections to search for that dataset. A `defaultdict` is recommended for the case where the same collections should be used for most datasets. skipExisting : `bool`, optional If `True` (default), a Quantum is not created if all its outputs already exist. """ for task in self.tasks: for quantumDataId in task.dataIds: # Identify the (regular) inputs that correspond to the Quantum # with this data ID. These are those whose data IDs have the # same values for all dimensions they have in common. # We do this data IDs expanded to include implied dimensions, # which is why _DatasetScaffolding.dimensions is thus expanded # even though DatasetType.dimensions is not. inputs = NamedKeyDict() for datasetType, scaffolding in task.inputs.items(): inputs[datasetType] = [ref for ref, dataId in zip(scaffolding.refs, scaffolding.dataIds) if quantumDataId.matches(dataId)] # Same for outputs. outputs = NamedKeyDict() allOutputsPresent = True for datasetType, scaffolding in task.outputs.items(): outputs[datasetType] = [] for ref, dataId in zip(scaffolding.refs, scaffolding.dataIds): if quantumDataId.matches(dataId): if ref.id is None: allOutputsPresent = False else: assert skipExisting, "Existing outputs should have already been identified." if not allOutputsPresent: raise OutputExistsError(f"Output {datasetType.name} with data ID " f"{dataId} already exists, but other outputs " f"for task with label {task.taskDef.label} " f"and data ID {quantumDataId} do not.") outputs[datasetType].append(ref) if allOutputsPresent and skipExisting: continue # Look up prerequisite datasets in the input collection(s). # These may have dimensions that extend beyond those we queried # for originally, because we want to permit those data ID # values to differ across quanta and dataset types. # For example, the same quantum may have a flat and bias with # a different calibration_label, or a refcat with a skypix # value that overlaps the quantum's data ID's region, but not # the user expression used for the initial query. for datasetType, scaffolding in task.prerequisites.items(): refs = list( registry.queryDatasets( datasetType, collections=inputCollections[datasetType.name], dataId=quantumDataId, deduplicate=True, expand=True, ) ) inputs[datasetType] = refs task.addQuantum( Quantum( taskName=task.taskDef.taskName, taskClass=task.taskDef.taskClass, dataId=quantumDataId, initInputs=task.initInputs.unpackRefs(), predictedInputs=inputs, outputs=outputs, ) )
def __init__(self, iterable=None): list.__init__(self, iterable or []) self.initInputs = NamedKeyDict() self.initIntermediates = NamedKeyDict() self.initOutputs = NamedKeyDict()