def write_fits(self, filename): """Write this object to a file. Parameters ---------- filename : `str` Name of file to write. """ # Create primary HDU with global metadata. metadata = PropertyList() metadata["HAS_DEFAULT"] = self.default_extended_psf is not None if self.focal_plane_regions: metadata["HAS_REGIONS"] = True metadata["REGION_NAMES"] = list(self.focal_plane_regions.keys()) for region, e_psf_region in self.focal_plane_regions.items(): metadata[region] = e_psf_region.detector_list else: metadata["HAS_REGIONS"] = False fits_primary = afwFits.Fits(filename, "w") fits_primary.createEmpty() fits_primary.writeMetadata(metadata) fits_primary.closeFile() # Write default extended PSF. if self.default_extended_psf is not None: default_hdu_metadata = PropertyList() default_hdu_metadata.update({ "REGION": "DEFAULT", "EXTNAME": "IMAGE" }) self.default_extended_psf.image.writeFits( filename, metadata=default_hdu_metadata, mode="a") default_hdu_metadata.update({ "REGION": "DEFAULT", "EXTNAME": "MASK" }) self.default_extended_psf.mask.writeFits( filename, metadata=default_hdu_metadata, mode="a") # Write extended PSF for each focal plane region. for j, (region, e_psf_region) in enumerate(self.focal_plane_regions.items()): metadata = PropertyList() metadata.update({"REGION": region, "EXTNAME": "IMAGE"}) e_psf_region.extended_psf_image.image.writeFits(filename, metadata=metadata, mode="a") metadata.update({"REGION": region, "EXTNAME": "MASK"}) e_psf_region.extended_psf_image.mask.writeFits(filename, metadata=metadata, mode="a")
def writeFits(filename, stamp_ims, metadata, write_mask, write_variance): """Write a single FITS file containing all stamps. Parameters ---------- filename : `str` A string indicating the output filename stamps_ims : iterable of `lsst.afw.image.MaskedImageF` An iterable of masked images metadata : `PropertyList` A collection of key, value metadata pairs to be written to the primary header write_mask : `bool` Write the mask data to the output file? write_variance : `bool` Write the variance data to the output file? """ metadata['HAS_MASK'] = write_mask metadata['HAS_VARIANCE'] = write_variance metadata['N_STAMPS'] = len(stamp_ims) # create primary HDU with global metadata fitsPrimary = afwFits.Fits(filename, "w") fitsPrimary.createEmpty() fitsPrimary.writeMetadata(metadata) fitsPrimary.closeFile() # add all pixel data optionally writing mask and variance information for i, stamp in enumerate(stamp_ims): metadata = PropertyList() # EXTVER should be 1-based, the index from enumerate is 0-based metadata.update({'EXTVER': i + 1, 'EXTNAME': 'IMAGE'}) stamp.getImage().writeFits(filename, metadata=metadata, mode='a') if write_mask: metadata = PropertyList() metadata.update({'EXTVER': i + 1, 'EXTNAME': 'MASK'}) stamp.getMask().writeFits(filename, metadata=metadata, mode='a') if write_variance: metadata = PropertyList() metadata.update({'EXTVER': i + 1, 'EXTNAME': 'VARIANCE'}) stamp.getVariance().writeFits(filename, metadata=metadata, mode='a') return None
def writeFits(filename, stamps, metadata, type_name, write_mask, write_variance, write_archive=False): """Write a single FITS file containing all stamps. Parameters ---------- filename : `str` A string indicating the output filename stamps : iterable of `BaseStamp` An iterable of Stamp objects metadata : `PropertyList` A collection of key, value metadata pairs to be written to the primary header type_name : `str` Python type name of the StampsBase subclass to use write_mask : `bool` Write the mask data to the output file? write_variance : `bool` Write the variance data to the output file? write_archive : `bool`, optional Write an archive to store Persistables along with each stamp? Default: ``False``. """ metadata['HAS_MASK'] = write_mask metadata['HAS_VARIANCE'] = write_variance metadata['HAS_ARCHIVE'] = write_archive metadata['N_STAMPS'] = len(stamps) metadata['STAMPCLS'] = type_name # Record version number in case of future code changes metadata['VERSION'] = 1 # create primary HDU with global metadata fitsFile = afwFits.Fits(filename, "w") fitsFile.createEmpty() # Store Persistables in an OutputArchive and write it if write_archive: oa = afwTable.io.OutputArchive() archive_ids = [oa.put(stamp.archive_element) for stamp in stamps] metadata["ARCHIVE_IDS"] = archive_ids fitsFile.writeMetadata(metadata) oa.writeFits(fitsFile) else: fitsFile.writeMetadata(metadata) fitsFile.closeFile() # add all pixel data optionally writing mask and variance information for i, stamp in enumerate(stamps): metadata = PropertyList() # EXTVER should be 1-based, the index from enumerate is 0-based metadata.update({'EXTVER': i + 1, 'EXTNAME': 'IMAGE'}) stamp.stamp_im.getImage().writeFits(filename, metadata=metadata, mode='a') if write_mask: metadata = PropertyList() metadata.update({'EXTVER': i + 1, 'EXTNAME': 'MASK'}) stamp.stamp_im.getMask().writeFits(filename, metadata=metadata, mode='a') if write_variance: metadata = PropertyList() metadata.update({'EXTVER': i + 1, 'EXTNAME': 'VARIANCE'}) stamp.stamp_im.getVariance().writeFits(filename, metadata=metadata, mode='a') return None
class IsrCalib(abc.ABC): """Generic calibration type. Subclasses must implement the toDict, fromDict, toTable, fromTable methods that allow the calibration information to be converted from dictionaries and afw tables. This will allow the calibration to be persisted using the base class read/write methods. The validate method is intended to provide a common way to check that the calibration is valid (internally consistent) and appropriate (usable with the intended data). The apply method is intended to allow the calibration to be applied in a consistent manner. Parameters ---------- camera : `lsst.afw.cameraGeom.Camera`, optional Camera to extract metadata from. detector : `lsst.afw.cameraGeom.Detector`, optional Detector to extract metadata from. log : `logging.Logger`, optional Log for messages. """ _OBSTYPE = "generic" _SCHEMA = "NO SCHEMA" _VERSION = 0 def __init__(self, camera=None, detector=None, log=None, **kwargs): self._instrument = None self._raftName = None self._slotName = None self._detectorName = None self._detectorSerial = None self._detectorId = None self._filter = None self._calibId = None self._metadata = PropertyList() self.setMetadata(PropertyList()) self.calibInfoFromDict(kwargs) # Define the required attributes for this calibration. self.requiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"]) self.requiredAttributes.update([ "_instrument", "_raftName", "_slotName", "_detectorName", "_detectorSerial", "_detectorId", "_filter", "_calibId", "_metadata" ]) self.log = log if log else logging.getLogger(__name__) if detector: self.fromDetector(detector) self.updateMetadata(camera=camera, detector=detector) def __str__(self): return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )" def __eq__(self, other): """Calibration equivalence. Running ``calib.log.setLevel(0)`` enables debug statements to identify problematic fields. """ if not isinstance(other, self.__class__): self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__) return False for attr in self._requiredAttributes: attrSelf = getattr(self, attr) attrOther = getattr(other, attr) if isinstance(attrSelf, dict): # Dictionary of arrays. if attrSelf.keys() != attrOther.keys(): self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys()) return False for key in attrSelf: if not np.allclose( attrSelf[key], attrOther[key], equal_nan=True): self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key]) return False elif isinstance(attrSelf, np.ndarray): # Bare array. if not np.allclose(attrSelf, attrOther, equal_nan=True): self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther) return False elif type(attrSelf) != type(attrOther): if set([attrSelf, attrOther]) == set([None, ""]): # Fits converts None to "", but None is not "". continue self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther), attrSelf, attrOther) return False else: if attrSelf != attrOther: self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther) return False return True @property def requiredAttributes(self): return self._requiredAttributes @requiredAttributes.setter def requiredAttributes(self, value): self._requiredAttributes = value def getMetadata(self): """Retrieve metadata associated with this calibration. Returns ------- meta : `lsst.daf.base.PropertyList` Metadata. The returned `~lsst.daf.base.PropertyList` can be modified by the caller and the changes will be written to external files. """ return self._metadata def setMetadata(self, metadata): """Store a copy of the supplied metadata with this calibration. Parameters ---------- metadata : `lsst.daf.base.PropertyList` Metadata to associate with the calibration. Will be copied and overwrite existing metadata. """ if metadata is not None: self._metadata.update(metadata) # Ensure that we have the obs type required by calibration ingest self._metadata["OBSTYPE"] = self._OBSTYPE self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION if isinstance(metadata, dict): self.calibInfoFromDict(metadata) elif isinstance(metadata, PropertyList): self.calibInfoFromDict(metadata.toDict()) def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs): """Update metadata keywords with new values. Parameters ---------- camera : `lsst.afw.cameraGeom.Camera`, optional Reference camera to use to set _instrument field. detector : `lsst.afw.cameraGeom.Detector`, optional Reference detector to use to set _detector* fields. filterName : `str`, optional Filter name to assign to this calibration. setCalibId : `bool`, optional Construct the _calibId field from other fields. setCalibInfo : `bool`, optional Set calibration parameters from metadata. setDate : `bool`, optional Ensure the metadata CALIBDATE fields are set to the current datetime. kwargs : `dict` or `collections.abc.Mapping`, optional Set of key=value pairs to assign to the metadata. """ mdOriginal = self.getMetadata() mdSupplemental = dict() for k, v in kwargs.items(): if isinstance(v, fits.card.Undefined): kwargs[k] = None if setCalibInfo: self.calibInfoFromDict(kwargs) if camera: self._instrument = camera.getName() if detector: self._detectorName = detector.getName() self._detectorSerial = detector.getSerial() self._detectorId = detector.getId() if "_" in self._detectorName: (self._raftName, self._slotName) = self._detectorName.split("_") if filterName: # TOD0 DM-28093: I think this whole comment can go away, if we # always use physicalLabel everywhere in ip_isr. # If set via: # exposure.getInfo().getFilter().getName() # then this will hold the abstract filter. self._filter = filterName if setDate: date = datetime.datetime.now() mdSupplemental["CALIBDATE"] = date.isoformat() mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat() mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat() if setCalibId: values = [] values.append( f"instrument={self._instrument}") if self._instrument else None values.append( f"raftName={self._raftName}") if self._raftName else None values.append(f"detectorName={self._detectorName}" ) if self._detectorName else None values.append( f"detector={self._detectorId}") if self._detectorId else None values.append(f"filter={self._filter}") if self._filter else None calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None)) values.append(f"calibDate={calibDate}") if calibDate else None self._calibId = " ".join(values) self._metadata[ "INSTRUME"] = self._instrument if self._instrument else None self._metadata["RAFTNAME"] = self._raftName if self._raftName else None self._metadata["SLOTNAME"] = self._slotName if self._slotName else None self._metadata["DETECTOR"] = self._detectorId self._metadata[ "DET_NAME"] = self._detectorName if self._detectorName else None self._metadata[ "DET_SER"] = self._detectorSerial if self._detectorSerial else None self._metadata["FILTER"] = self._filter if self._filter else None self._metadata["CALIB_ID"] = self._calibId if self._calibId else None self._metadata["CALIBCLS"] = get_full_type_name(self) mdSupplemental.update(kwargs) mdOriginal.update(mdSupplemental) def calibInfoFromDict(self, dictionary): """Handle common keywords. This isn't an ideal solution, but until all calibrations expect to find everything in the metadata, they still need to search through dictionaries. Parameters ---------- dictionary : `dict` or `lsst.daf.base.PropertyList` Source for the common keywords. Raises ------ RuntimeError : Raised if the dictionary does not match the expected OBSTYPE. """ def search(haystack, needles): """Search dictionary 'haystack' for an entry in 'needles' """ test = [haystack.get(x) for x in needles] test = set([x for x in test if x is not None]) if len(test) == 0: if "metadata" in haystack: return search(haystack["metadata"], needles) else: return None elif len(test) == 1: value = list(test)[0] if value == "": return None else: return value else: raise ValueError( f"Too many values found: {len(test)} {test} {needles}") if "metadata" in dictionary: metadata = dictionary["metadata"] if self._OBSTYPE != metadata["OBSTYPE"]: raise RuntimeError( f"Incorrect calibration supplied. Expected {self._OBSTYPE}, " f"found {metadata['OBSTYPE']}") self._instrument = search(dictionary, ["INSTRUME", "instrument"]) self._raftName = search(dictionary, ["RAFTNAME"]) self._slotName = search(dictionary, ["SLOTNAME"]) self._detectorId = search(dictionary, ["DETECTOR", "detectorId"]) self._detectorName = search( dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"]) self._detectorSerial = search( dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"]) self._filter = search(dictionary, ["FILTER", "filterName"]) self._calibId = search(dictionary, ["CALIB_ID"]) @classmethod def determineCalibClass(cls, metadata, message): """Attempt to find calibration class in metadata. Parameters ---------- metadata : `dict` or `lsst.daf.base.PropertyList` Metadata possibly containing a calibration class entry. message : `str` Message to include in any errors. Returns ------- calibClass : `object` The class to use to read the file contents. Should be an `lsst.ip.isr.IsrCalib` subclass. Raises ------ ValueError : Raised if the resulting calibClass is the base `lsst.ip.isr.IsrClass` (which does not implement the content methods). """ calibClassName = metadata.get("CALIBCLS") calibClass = doImport( calibClassName) if calibClassName is not None else cls if calibClass is IsrCalib: raise ValueError( f"Cannot use base class to read calibration data: {message}") return calibClass @classmethod def readText(cls, filename, **kwargs): """Read calibration representation from a yaml/ecsv file. Parameters ---------- filename : `str` Name of the file containing the calibration definition. kwargs : `dict` or collections.abc.Mapping`, optional Set of key=value pairs to pass to the ``fromDict`` or ``fromTable`` methods. Returns ------- calib : `~lsst.ip.isr.IsrCalibType` Calibration class. Raises ------ RuntimeError : Raised if the filename does not end in ".ecsv" or ".yaml". """ if filename.endswith((".ecsv", ".ECSV")): data = Table.read(filename, format="ascii.ecsv") calibClass = cls.determineCalibClass(data.meta, "readText/ECSV") return calibClass.fromTable([data], **kwargs) elif filename.endswith((".yaml", ".YAML")): with open(filename, "r") as f: data = yaml.load(f, Loader=yaml.CLoader) calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML") return calibClass.fromDict(data, **kwargs) else: raise RuntimeError(f"Unknown filename extension: {filename}") def writeText(self, filename, format="auto"): """Write the calibration data to a text file. Parameters ---------- filename : `str` Name of the file to write. format : `str` Format to write the file as. Supported values are: ``"auto"`` : Determine filetype from filename. ``"yaml"`` : Write as yaml. ``"ecsv"`` : Write as ecsv. Returns ------- used : `str` The name of the file used to write the data. This may differ from the input if the format is explicitly chosen. Raises ------ RuntimeError : Raised if filename does not end in a known extension, or if all information cannot be written. Notes ----- The file is written to YAML/ECSV format and will include any associated metadata. """ if format == "yaml" or (format == "auto" and filename.lower().endswith( (".yaml", ".YAML"))): outDict = self.toDict() path, ext = os.path.splitext(filename) filename = path + ".yaml" with open(filename, "w") as f: yaml.dump(outDict, f) elif format == "ecsv" or (format == "auto" and filename.lower().endswith( (".ecsv", ".ECSV"))): tableList = self.toTable() if len(tableList) > 1: # ECSV doesn't support multiple tables per file, so we # can only write the first table. raise RuntimeError( f"Unable to persist {len(tableList)}tables in ECSV format." ) table = tableList[0] path, ext = os.path.splitext(filename) filename = path + ".ecsv" table.write(filename, format="ascii.ecsv") else: raise RuntimeError(f"Attempt to write to a file {filename} " "that does not end in '.yaml' or '.ecsv'") return filename @classmethod def readFits(cls, filename, **kwargs): """Read calibration data from a FITS file. Parameters ---------- filename : `str` Filename to read data from. kwargs : `dict` or collections.abc.Mapping`, optional Set of key=value pairs to pass to the ``fromTable`` method. Returns ------- calib : `lsst.ip.isr.IsrCalib` Calibration contained within the file. """ tableList = [] tableList.append(Table.read(filename, hdu=1)) extNum = 2 # Fits indices start at 1, we've read one already. keepTrying = True while keepTrying: with warnings.catch_warnings(): warnings.simplefilter("error") try: newTable = Table.read(filename, hdu=extNum) tableList.append(newTable) extNum += 1 except Exception: keepTrying = False for table in tableList: for k, v in table.meta.items(): if isinstance(v, fits.card.Undefined): table.meta[k] = None calibClass = cls.determineCalibClass(tableList[0].meta, "readFits") return calibClass.fromTable(tableList, **kwargs) def writeFits(self, filename): """Write calibration data to a FITS file. Parameters ---------- filename : `str` Filename to write data to. Returns ------- used : `str` The name of the file used to write the data. """ tableList = self.toTable() with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=Warning, module="astropy.io") astropyList = [fits.table_to_hdu(table) for table in tableList] astropyList.insert(0, fits.PrimaryHDU()) writer = fits.HDUList(astropyList) writer.writeto(filename, overwrite=True) return filename def fromDetector(self, detector): """Modify the calibration parameters to match the supplied detector. Parameters ---------- detector : `lsst.afw.cameraGeom.Detector` Detector to use to set parameters from. Raises ------ NotImplementedError This needs to be implemented by subclasses for each calibration type. """ raise NotImplementedError("Must be implemented by subclass.") @classmethod def fromDict(cls, dictionary, **kwargs): """Construct a calibration from a dictionary of properties. Must be implemented by the specific calibration subclasses. Parameters ---------- dictionary : `dict` Dictionary of properties. kwargs : `dict` or collections.abc.Mapping`, optional Set of key=value options. Returns ------ calib : `lsst.ip.isr.CalibType` Constructed calibration. Raises ------ NotImplementedError : Raised if not implemented. """ raise NotImplementedError("Must be implemented by subclass.") def toDict(self): """Return a dictionary containing the calibration properties. The dictionary should be able to be round-tripped through `fromDict`. Returns ------- dictionary : `dict` Dictionary of properties. Raises ------ NotImplementedError : Raised if not implemented. """ raise NotImplementedError("Must be implemented by subclass.") @classmethod def fromTable(cls, tableList, **kwargs): """Construct a calibration from a dictionary of properties. Must be implemented by the specific calibration subclasses. Parameters ---------- tableList : `list` [`lsst.afw.table.Table`] List of tables of properties. kwargs : `dict` or collections.abc.Mapping`, optional Set of key=value options. Returns ------ calib : `lsst.ip.isr.CalibType` Constructed calibration. Raises ------ NotImplementedError : Raised if not implemented. """ raise NotImplementedError("Must be implemented by subclass.") def toTable(self): """Return a list of tables containing the calibration properties. The table list should be able to be round-tripped through `fromDict`. Returns ------- tableList : `list` [`lsst.afw.table.Table`] List of tables of properties. Raises ------ NotImplementedError : Raised if not implemented. """ raise NotImplementedError("Must be implemented by subclass.") def validate(self, other=None): """Validate that this calibration is defined and can be used. Parameters ---------- other : `object`, optional Thing to validate against. Returns ------- valid : `bool` Returns true if the calibration is valid and appropriate. """ return False def apply(self, target): """Method to apply the calibration to the target object. Parameters ---------- target : `object` Thing to validate against. Returns ------- valid : `bool` Returns true if the calibration was applied correctly. Raises ------ NotImplementedError : Raised if not implemented. """ raise NotImplementedError("Must be implemented by subclass.")