def from_string(name: str, registry: Optional[Registry] = None, collection_prefix: Optional[str] = None) -> Instrument: """Return an instance from the short name or class name. If the instrument name is not qualified (does not contain a '.') and a butler registry is provided, this will attempt to load the instrument using `Instrument.fromName()`. Otherwise the instrument will be imported and instantiated. Parameters ---------- name : `str` The name or fully-qualified class name of an instrument. registry : `lsst.daf.butler.Registry`, optional Butler registry to query to find information about the instrument, by default `None`. collection_prefix : `str`, optional Prefix for collection names to use instead of the intrument's own name. This is primarily for use in simulated-data repositories, where the instrument name may not be necessary and/or sufficient to distinguish between collections. Returns ------- instrument : `Instrument` The instantiated instrument. Raises ------ RuntimeError Raised if the instrument can not be imported, instantiated, or obtained from the registry. TypeError Raised if the instrument is not a subclass of `~lsst.pipe.base.Instrument`. See Also -------- Instrument.fromName """ if "." not in name and registry is not None: try: instr = Instrument.fromName( name, registry, collection_prefix=collection_prefix) except Exception as err: raise RuntimeError( f"Could not get instrument from name: {name}. Failed with exception: {err}" ) from err else: try: instr_class = doImportType(name) except Exception as err: raise RuntimeError( f"Could not import instrument: {name}. Failed with exception: {err}" ) from err instr = instr_class(collection_prefix=collection_prefix) if not isinstance(instr, Instrument): raise TypeError(f"{name} is not an Instrument subclass.") return instr
def fromName(name: str, registry: Registry, collection_prefix: Optional[str] = None) -> Instrument: """Given an instrument name and a butler registry, retrieve a corresponding instantiated instrument object. Parameters ---------- name : `str` Name of the instrument (must match the return value of `getName`). registry : `lsst.daf.butler.Registry` Butler registry to query to find the information. collection_prefix : `str`, optional Prefix for collection names to use instead of the intrument's own name. This is primarily for use in simulated-data repositories, where the instrument name may not be necessary and/or sufficient to distinguish between collections. Returns ------- instrument : `Instrument` An instance of the relevant `Instrument`. Notes ----- The instrument must be registered in the corresponding butler. Raises ------ LookupError Raised if the instrument is not known to the supplied registry. ModuleNotFoundError Raised if the class could not be imported. This could mean that the relevant obs package has not been setup. TypeError Raised if the class name retrieved is not a string or the imported symbol is not an `Instrument` subclass. """ try: records = list( registry.queryDimensionRecords("instrument", instrument=name)) except DataIdError: records = None if not records: raise LookupError(f"No registered instrument with name '{name}'.") cls_name = records[0].class_name if not isinstance(cls_name, str): raise TypeError( f"Unexpected class name retrieved from {name} instrument dimension (got {cls_name})" ) instrument_cls: type = doImportType(cls_name) if not issubclass(instrument_cls, Instrument): raise TypeError( f"{instrument_cls!r}, obtained from importing {cls_name}, is not an Instrument subclass." ) return instrument_cls(collection_prefix=collection_prefix)
def importAll(registry: Registry) -> None: """Import all the instruments known to this registry. This will ensure that all metadata translators have been registered. Parameters ---------- registry : `lsst.daf.butler.Registry` Butler registry to query to find the information. Notes ----- It is allowed for a particular instrument class to fail on import. This might simply indicate that a particular obs package has not been setup. """ records = list(registry.queryDimensionRecords("instrument")) for record in records: cls = record.class_name try: doImportType(cls) except Exception: pass
def addInstrumentOverride(self, instrument: str, task_name: str) -> None: """Apply any overrides that an instrument has for a task Parameters ---------- instrument: str A string containing the fully qualified name of an instrument from which configs should be loaded and applied task_name: str The _DefaultName of a task associated with a config, used to look up overrides from the instrument. """ instrument_cls: type = doImportType(instrument) instrument_lib = instrument_cls() self._overrides.append( (OverrideTypes.Instrument, (instrument_lib, task_name)))
def populateButler( pipeline: Pipeline, butler: Butler, datasetTypes: Dict[Optional[str], List[str]] = None) -> None: """Populate data butler with data needed for test. Initializes data butler with a bunch of items: - registers dataset types which are defined by pipeline - create dimensions data for (instrument, detector) - adds datasets based on ``datasetTypes`` dictionary, if dictionary is missing then a single dataset with type "add_dataset0" is added All datasets added to butler have ``dataId={instrument=instrument, detector=0}`` where ``instrument`` is extracted from pipeline, "INSTR" is used if pipeline is missing instrument definition. Type of the dataset is guessed from dataset type name (assumes that pipeline is made of `AddTask` tasks). Parameters ---------- pipeline : `~lsst.pipe.base.Pipeline` Pipeline instance. butler : `~lsst.daf.butler.Butler` Data butler instance. datasetTypes : `dict` [ `str`, `list` ], optional Dictionary whose keys are collection names and values are lists of dataset type names. By default a single dataset of type "add_dataset0" is added to a ``butler.run`` collection. """ # Add dataset types to registry taskDefs = list(pipeline.toExpandedPipeline()) registerDatasetTypes(butler.registry, taskDefs) instrument = pipeline.getInstrument() if instrument is not None: instrument_class = doImportType(instrument) instrumentName = instrument_class.getName() else: instrumentName = "INSTR" # Add all needed dimensions to registry butler.registry.insertDimensionData("instrument", dict(name=instrumentName)) butler.registry.insertDimensionData( "detector", dict(instrument=instrumentName, id=0, full_name="det0")) taskDefMap = dict((taskDef.label, taskDef) for taskDef in taskDefs) # Add inputs to butler if not datasetTypes: datasetTypes = {None: ["add_dataset0"]} for run, dsTypes in datasetTypes.items(): if run is not None: butler.registry.registerRun(run) for dsType in dsTypes: if dsType == "packages": # Version is intentionally inconsistent. # Dict is convertible to Packages if Packages is installed. data: Any = {"python": "9.9.99"} butler.put(data, dsType, run=run) else: if dsType.endswith("_config"): # find a confing from matching task name or make a new one taskLabel, _, _ = dsType.rpartition("_") taskDef = taskDefMap.get(taskLabel) if taskDef is not None: data = taskDef.config else: data = AddTaskConfig() elif dsType.endswith("_metadata"): data = _TASK_FULL_METADATA_TYPE() elif dsType.endswith("_log"): data = ButlerLogRecords.from_records([]) else: data = numpy.array([0.0, 1.0, 2.0, 5.0]) butler.put(data, dsType, run=run, instrument=instrumentName, detector=0)
def testDoImportType(self): with self.assertRaises(TypeError): doImportType("lsst.utils") c = doImportType("lsst.utils.tests.TestCase") self.assertEqual(c, lsst.utils.tests.TestCase)
def constructGraph(self, nodes: set[uuid.UUID], _readBytes: Callable[[int, int], bytes], universe: DimensionUniverse) -> QuantumGraph: # need to import here to avoid cyclic imports from . import QuantumGraph graph = nx.DiGraph() loadedTaskDef: Dict[str, TaskDef] = {} container = {} datasetDict = _DatasetTracker[DatasetTypeName, TaskDef](createInverse=True) taskToQuantumNode: DefaultDict[TaskDef, Set[QuantumNode]] = defaultdict(set) recontitutedDimensions: Dict[int, Tuple[str, DimensionRecord]] = {} for node in nodes: start, stop = self.infoMappings.map[node]["bytes"] start, stop = start + self.headerSize, stop + self.headerSize # Read in the bytes corresponding to the node to load and # decompress it dump = json.loads(lzma.decompress(_readBytes(start, stop))) # Turn the json back into the pydandtic model nodeDeserialized = SerializedQuantumNode.direct(**dump) # attach the dictionary of dimension records to the pydandtic model # these are stored seperately because the are stored over and over # and this saves a lot of space and time. nodeDeserialized.quantum.dimensionRecords = self.infoMappings.dimensionRecords # get the label for the current task nodeTaskLabel = nodeDeserialized.taskLabel if nodeTaskLabel not in loadedTaskDef: # Get the byte ranges corresponding to this taskDef start, stop = self.infoMappings.taskDefMap[nodeTaskLabel][ "bytes"] start, stop = start + self.headerSize, stop + self.headerSize # bytes are compressed, so decompress them taskDefDump = json.loads( lzma.decompress(_readBytes(start, stop))) taskClass: Type[PipelineTask] = doImportType( taskDefDump["taskName"]) config: Config = taskClass.ConfigClass() config.loadFromStream(taskDefDump["config"]) # Rebuild TaskDef recreatedTaskDef = TaskDef( taskName=taskDefDump["taskName"], taskClass=taskClass, config=config, label=taskDefDump["label"], ) loadedTaskDef[nodeTaskLabel] = recreatedTaskDef # rebuild the mappings that associate dataset type names with # TaskDefs for _, input in self.infoMappings.taskDefMap[nodeTaskLabel][ "inputs"]: datasetDict.addConsumer(DatasetTypeName(input), recreatedTaskDef) added = set() for outputConnection in self.infoMappings.taskDefMap[ nodeTaskLabel]["outputs"]: typeName = outputConnection[1] if typeName not in added: added.add(typeName) datasetDict.addProducer(DatasetTypeName(typeName), recreatedTaskDef) # reconstitute the node, passing in the dictionaries for the # loaded TaskDefs and dimension records. These are used to ensure # that each unique record is only loaded once qnode = QuantumNode.from_simple(nodeDeserialized, loadedTaskDef, universe, recontitutedDimensions) container[qnode.nodeId] = qnode taskToQuantumNode[loadedTaskDef[nodeTaskLabel]].add(qnode) # recreate the relations between each node from stored info graph.add_node(qnode) for id in self.infoMappings.map[qnode.nodeId]["inputs"]: # uuid is stored as a string, turn it back into a uuid id = uuid.UUID(id) # if the id is not yet in the container, dont make a connection # this is not an issue, because once it is, that id will add # the reverse connection if id in container: graph.add_edge(container[id], qnode) for id in self.infoMappings.map[qnode.nodeId]["outputs"]: # uuid is stored as a string, turn it back into a uuid id = uuid.UUID(id) # if the id is not yet in the container, dont make a connection # this is not an issue, because once it is, that id will add # the reverse connection if id in container: graph.add_edge(qnode, container[id]) newGraph = object.__new__(QuantumGraph) newGraph._metadata = self.infoMappings.metadata newGraph._buildId = self.infoMappings._buildId newGraph._datasetDict = datasetDict newGraph._nodeIdMap = container newGraph._count = len(nodes) newGraph._taskToQuantumNode = dict(taskToQuantumNode.items()) newGraph._taskGraph = datasetDict.makeNetworkXGraph() newGraph._connectedQuanta = graph return newGraph