def from_string(name: str,
registry: Optional[Registry] = None,
collection_prefix: Optional[str] = None) -> Instrument:
"""Return an instance from the short name or class name.
If the instrument name is not qualified (does not contain a '.') and a
butler registry is provided, this will attempt to load the instrument
using `Instrument.fromName()`. Otherwise the instrument will be
imported and instantiated.
Parameters
----------
name : `str`
The name or fully-qualified class name of an instrument.
registry : `lsst.daf.butler.Registry`, optional
Butler registry to query to find information about the instrument,
by default `None`.
collection_prefix : `str`, optional
Prefix for collection names to use instead of the intrument's own
name. This is primarily for use in simulated-data repositories,
where the instrument name may not be necessary and/or sufficient
to distinguish between collections.
Returns
-------
instrument : `Instrument`
The instantiated instrument.
Raises
------
RuntimeError
Raised if the instrument can not be imported, instantiated, or
obtained from the registry.
TypeError
Raised if the instrument is not a subclass of
`~lsst.pipe.base.Instrument`.
See Also
--------
Instrument.fromName
"""
if "." not in name and registry is not None:
try:
instr = Instrument.fromName(
name, registry, collection_prefix=collection_prefix)
except Exception as err:
raise RuntimeError(
f"Could not get instrument from name: {name}. Failed with exception: {err}"
) from err
else:
try:
instr_class = doImportType(name)
except Exception as err:
raise RuntimeError(
f"Could not import instrument: {name}. Failed with exception: {err}"
) from err
instr = instr_class(collection_prefix=collection_prefix)
if not isinstance(instr, Instrument):
raise TypeError(f"{name} is not an Instrument subclass.")
return instr
def fromName(name: str,
registry: Registry,
collection_prefix: Optional[str] = None) -> Instrument:
"""Given an instrument name and a butler registry, retrieve a
corresponding instantiated instrument object.
Parameters
----------
name : `str`
Name of the instrument (must match the return value of `getName`).
registry : `lsst.daf.butler.Registry`
Butler registry to query to find the information.
collection_prefix : `str`, optional
Prefix for collection names to use instead of the intrument's own
name. This is primarily for use in simulated-data repositories,
where the instrument name may not be necessary and/or sufficient to
distinguish between collections.
Returns
-------
instrument : `Instrument`
An instance of the relevant `Instrument`.
Notes
-----
The instrument must be registered in the corresponding butler.
Raises
------
LookupError
Raised if the instrument is not known to the supplied registry.
ModuleNotFoundError
Raised if the class could not be imported. This could mean
that the relevant obs package has not been setup.
TypeError
Raised if the class name retrieved is not a string or the imported
symbol is not an `Instrument` subclass.
"""
try:
records = list(
registry.queryDimensionRecords("instrument", instrument=name))
except DataIdError:
records = None
if not records:
raise LookupError(f"No registered instrument with name '{name}'.")
cls_name = records[0].class_name
if not isinstance(cls_name, str):
raise TypeError(
f"Unexpected class name retrieved from {name} instrument dimension (got {cls_name})"
)
instrument_cls: type = doImportType(cls_name)
if not issubclass(instrument_cls, Instrument):
raise TypeError(
f"{instrument_cls!r}, obtained from importing {cls_name}, is not an Instrument subclass."
)
return instrument_cls(collection_prefix=collection_prefix)
def importAll(registry: Registry) -> None:
"""Import all the instruments known to this registry.
This will ensure that all metadata translators have been registered.
Parameters
----------
registry : `lsst.daf.butler.Registry`
Butler registry to query to find the information.
Notes
-----
It is allowed for a particular instrument class to fail on import.
This might simply indicate that a particular obs package has
not been setup.
"""
records = list(registry.queryDimensionRecords("instrument"))
for record in records:
cls = record.class_name
try:
doImportType(cls)
except Exception:
pass
def addInstrumentOverride(self, instrument: str, task_name: str) -> None:
"""Apply any overrides that an instrument has for a task
Parameters
----------
instrument: str
A string containing the fully qualified name of an instrument from
which configs should be loaded and applied
task_name: str
The _DefaultName of a task associated with a config, used to look
up overrides from the instrument.
"""
instrument_cls: type = doImportType(instrument)
instrument_lib = instrument_cls()
self._overrides.append(
(OverrideTypes.Instrument, (instrument_lib, task_name)))
def populateButler(
pipeline: Pipeline,
butler: Butler,
datasetTypes: Dict[Optional[str], List[str]] = None) -> None:
"""Populate data butler with data needed for test.
Initializes data butler with a bunch of items:
- registers dataset types which are defined by pipeline
- create dimensions data for (instrument, detector)
- adds datasets based on ``datasetTypes`` dictionary, if dictionary is
missing then a single dataset with type "add_dataset0" is added
All datasets added to butler have ``dataId={instrument=instrument,
detector=0}`` where ``instrument`` is extracted from pipeline, "INSTR" is
used if pipeline is missing instrument definition. Type of the dataset is
guessed from dataset type name (assumes that pipeline is made of `AddTask`
tasks).
Parameters
----------
pipeline : `~lsst.pipe.base.Pipeline`
Pipeline instance.
butler : `~lsst.daf.butler.Butler`
Data butler instance.
datasetTypes : `dict` [ `str`, `list` ], optional
Dictionary whose keys are collection names and values are lists of
dataset type names. By default a single dataset of type "add_dataset0"
is added to a ``butler.run`` collection.
"""
# Add dataset types to registry
taskDefs = list(pipeline.toExpandedPipeline())
registerDatasetTypes(butler.registry, taskDefs)
instrument = pipeline.getInstrument()
if instrument is not None:
instrument_class = doImportType(instrument)
instrumentName = instrument_class.getName()
else:
instrumentName = "INSTR"
# Add all needed dimensions to registry
butler.registry.insertDimensionData("instrument",
dict(name=instrumentName))
butler.registry.insertDimensionData(
"detector", dict(instrument=instrumentName, id=0, full_name="det0"))
taskDefMap = dict((taskDef.label, taskDef) for taskDef in taskDefs)
# Add inputs to butler
if not datasetTypes:
datasetTypes = {None: ["add_dataset0"]}
for run, dsTypes in datasetTypes.items():
if run is not None:
butler.registry.registerRun(run)
for dsType in dsTypes:
if dsType == "packages":
# Version is intentionally inconsistent.
# Dict is convertible to Packages if Packages is installed.
data: Any = {"python": "9.9.99"}
butler.put(data, dsType, run=run)
else:
if dsType.endswith("_config"):
# find a confing from matching task name or make a new one
taskLabel, _, _ = dsType.rpartition("_")
taskDef = taskDefMap.get(taskLabel)
if taskDef is not None:
data = taskDef.config
else:
data = AddTaskConfig()
elif dsType.endswith("_metadata"):
data = _TASK_FULL_METADATA_TYPE()
elif dsType.endswith("_log"):
data = ButlerLogRecords.from_records([])
else:
data = numpy.array([0.0, 1.0, 2.0, 5.0])
butler.put(data,
dsType,
run=run,
instrument=instrumentName,
detector=0)
def testDoImportType(self):
with self.assertRaises(TypeError):
doImportType("lsst.utils")
c = doImportType("lsst.utils.tests.TestCase")
self.assertEqual(c, lsst.utils.tests.TestCase)
def constructGraph(self, nodes: set[uuid.UUID],
_readBytes: Callable[[int, int], bytes],
universe: DimensionUniverse) -> QuantumGraph:
# need to import here to avoid cyclic imports
from . import QuantumGraph
graph = nx.DiGraph()
loadedTaskDef: Dict[str, TaskDef] = {}
container = {}
datasetDict = _DatasetTracker[DatasetTypeName,
TaskDef](createInverse=True)
taskToQuantumNode: DefaultDict[TaskDef,
Set[QuantumNode]] = defaultdict(set)
recontitutedDimensions: Dict[int, Tuple[str, DimensionRecord]] = {}
for node in nodes:
start, stop = self.infoMappings.map[node]["bytes"]
start, stop = start + self.headerSize, stop + self.headerSize
# Read in the bytes corresponding to the node to load and
# decompress it
dump = json.loads(lzma.decompress(_readBytes(start, stop)))
# Turn the json back into the pydandtic model
nodeDeserialized = SerializedQuantumNode.direct(**dump)
# attach the dictionary of dimension records to the pydandtic model
# these are stored seperately because the are stored over and over
# and this saves a lot of space and time.
nodeDeserialized.quantum.dimensionRecords = self.infoMappings.dimensionRecords
# get the label for the current task
nodeTaskLabel = nodeDeserialized.taskLabel
if nodeTaskLabel not in loadedTaskDef:
# Get the byte ranges corresponding to this taskDef
start, stop = self.infoMappings.taskDefMap[nodeTaskLabel][
"bytes"]
start, stop = start + self.headerSize, stop + self.headerSize
# bytes are compressed, so decompress them
taskDefDump = json.loads(
lzma.decompress(_readBytes(start, stop)))
taskClass: Type[PipelineTask] = doImportType(
taskDefDump["taskName"])
config: Config = taskClass.ConfigClass()
config.loadFromStream(taskDefDump["config"])
# Rebuild TaskDef
recreatedTaskDef = TaskDef(
taskName=taskDefDump["taskName"],
taskClass=taskClass,
config=config,
label=taskDefDump["label"],
)
loadedTaskDef[nodeTaskLabel] = recreatedTaskDef
# rebuild the mappings that associate dataset type names with
# TaskDefs
for _, input in self.infoMappings.taskDefMap[nodeTaskLabel][
"inputs"]:
datasetDict.addConsumer(DatasetTypeName(input),
recreatedTaskDef)
added = set()
for outputConnection in self.infoMappings.taskDefMap[
nodeTaskLabel]["outputs"]:
typeName = outputConnection[1]
if typeName not in added:
added.add(typeName)
datasetDict.addProducer(DatasetTypeName(typeName),
recreatedTaskDef)
# reconstitute the node, passing in the dictionaries for the
# loaded TaskDefs and dimension records. These are used to ensure
# that each unique record is only loaded once
qnode = QuantumNode.from_simple(nodeDeserialized, loadedTaskDef,
universe, recontitutedDimensions)
container[qnode.nodeId] = qnode
taskToQuantumNode[loadedTaskDef[nodeTaskLabel]].add(qnode)
# recreate the relations between each node from stored info
graph.add_node(qnode)
for id in self.infoMappings.map[qnode.nodeId]["inputs"]:
# uuid is stored as a string, turn it back into a uuid
id = uuid.UUID(id)
# if the id is not yet in the container, dont make a connection
# this is not an issue, because once it is, that id will add
# the reverse connection
if id in container:
graph.add_edge(container[id], qnode)
for id in self.infoMappings.map[qnode.nodeId]["outputs"]:
# uuid is stored as a string, turn it back into a uuid
id = uuid.UUID(id)
# if the id is not yet in the container, dont make a connection
# this is not an issue, because once it is, that id will add
# the reverse connection
if id in container:
graph.add_edge(qnode, container[id])
newGraph = object.__new__(QuantumGraph)
newGraph._metadata = self.infoMappings.metadata
newGraph._buildId = self.infoMappings._buildId
newGraph._datasetDict = datasetDict
newGraph._nodeIdMap = container
newGraph._count = len(nodes)
newGraph._taskToQuantumNode = dict(taskToQuantumNode.items())
newGraph._taskGraph = datasetDict.makeNetworkXGraph()
newGraph._connectedQuanta = graph
return newGraph