def makeMinimalIsrConfig(self): """Return an IsrConfig with all boolean flags disabled""" isrConfig = IsrTask.ConfigClass() for name in isrConfig: if name.startswith("do"): setattr(isrConfig, name, False) return isrConfig
def runIsr(): '''Run the task to do ISR on a ccd''' #Create the isr task with modified config isrConfig = IsrTask.ConfigClass() isrConfig.doBias = False #We didn't make a zero frame isrConfig.doDark = True isrConfig.doFlat = True isrConfig.doFringe = False #There is no fringe frame for this example isrConfig.assembleCcd.setGain = False isrTask = IsrTask(config=isrConfig) #Make raw, flat and dark exposures DARKVAL = 2. #e-/sec OSCAN = 1000. #DN GRADIENT = .10 EXPTIME = 15 #seconds DARKEXPTIME = 40. #seconds darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME) flatExposure = exampleUtils.makeFlat(GRADIENT) rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME) output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure) return output.exposure
def testTaskAPI(self): """Test that the Tasks work Checks both MeasureCrosstalkTask and the CrosstalkTask. """ # make exposure available to NullIsrTask # without NullIsrTask's `self` hiding this test class's `self` exposure = self.exposure class NullIsrTask(IsrTask): def runDataRef(self, dataRef): return Struct(exposure=exposure) config = MeasureCrosstalkTask.ConfigClass() config.isr.retarget(NullIsrTask) config.threshold = self.value - 1 measure = MeasureCrosstalkTask(config=config) fakeDataRef = Struct(dataId={'fake': 1}) coeff, coeffErr, coeffNum = measure.reduce( [measure.runDataRef(fakeDataRef)]) self.checkCoefficients(coeff, coeffErr, coeffNum) config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) isr.crosstalk.run(self.exposure) self.checkSubtracted(self.exposure)
def processExposure(exposure, bias=None, defects=None, repair=False, repo=None): from lsst.ip.isr import IsrTask import lsst.daf.persistence as dafPersist import time config = IsrTask.ConfigClass() if (bias == None or defects == None) and repo == None: raise (AttributeError( 'Repo keyword not set and is required to find bias/defect frames')) else: butler = dafPersist.Butler(repo) if False: config.overscanFitType = "AKIMA_SPLINE" config.overscanOrder = 5 else: config.overscanFitType = "POLY" config.overscanOrder = 3 config.doBias = True if config.doBias == True and bias == None: bias = butler.get( 'bias', visit=exposure.getInfo().getVisitInfo().getExposureId(), ) config.doDark = False config.doFringe = False config.doFlat = False config.doLinearize = False # TODO: should be able to run this but there are issues with defects in the stack at the moment config.doDefect = True if repair == True else False if (config.doDefect == True and bias == None): butler = dafPersist.Butler(repo) defects = butler.get( 'defects', visit=exposure.getInfo().getVisitInfo().getExposureId(), ) config.doAddDistortionModel = False config.doSaturationInterpolation = False config.overscanNumLeadingColumnsToSkip = 20 isrTask = IsrTask(config=config) # Run ISR start = time.time() isr_corrected_exposure = isrTask.run(exposure, bias=bias, defects=defects).exposure end = time.time() print('Time to perform image ISR was {0:2f} [s]'.format(end - start)) return (isr_corrected_exposure)
def testTaskAPI(self): """Test that the Tasks work Checks both MeasureCrosstalkTask and the CrosstalkTask. """ # make exposure available to NullIsrTask # without NullIsrTask's `self` hiding this test class's `self` exposure = self.exposure class NullIsrTask(IsrTask): def runDataRef(self, dataRef): return Struct(exposure=exposure) coeff = np.array(self.crosstalk).transpose() config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(), coeffVector=coeff) isr.crosstalk.run(self.exposure, crosstalk=calib) self.checkSubtracted(self.exposure)
def test_interChip(self): """Test that passing an external exposure as the crosstalk source works. """ exposure = self.exposure ctSources = [self.ctSource] coeff = np.array(self.crosstalk).transpose() calib = CrosstalkCalib().fromDetector(exposure.getDetector(), coeffVector=coeff) # Now convert this into zero intra-chip, full inter-chip: calib.interChip['detector 2'] = coeff calib.coeffs = np.zeros_like(coeff) # Process and check as above config = IsrTask.ConfigClass() config.crosstalk.minPixelToMask = self.value - 1 config.crosstalk.crosstalkMaskPlane = self.crosstalkStr isr = IsrTask(config=config) isr.crosstalk.run(exposure, crosstalk=calib, crosstalkSources=ctSources) self.checkSubtracted(exposure)
def __init__(self, *, config=None): # programatically clone all of the connections from isrTask # settin minimum values to zero for everything except the ccdExposure super().__init__(config=IsrTask.ConfigClass( )) # need a dummy config, isn't used other than for ctor for name, connection in self.allConnections.items(): if hasattr(connection, 'minimum'): args = _getArgs(connection) if name != "ccdExposure": # need one input image always args['minimum'] = 0 newConnection = type(connection)(**args) self.allConnections[name] = newConnection setattr(self, name, newConnection) exposure = cT.Output( # called just "exposure" to mimic isrTask's return struct name="quickLookExp", doc="The quickLook output exposure.", storageClass="ExposureF", dimensions=("instrument", "exposure", "detector"), ) # set like this to make it explicit that the outputExposure # and the exposure are identical. The only reason there are two is for # API compatibility. self.outputExposure = exposure
def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, **kwargs): """Get the postIsr and cosmic-repaired image for this dataId. Parameters ---------- expIdOrDataId : `dict` The dataId extraIsrOptions : `dict`, optional extraIsrOptions is a dict of extra isr options applied to this image only. skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK! Skip doing cosmic ray repair for this image? Returns ------- exp : `lsst.afw.image.Exposure` The postIsr exposure """ dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs) try: exp = self.butler.get(self._datasetName, dataId=dataId) self.log.info("Found a ready-made quickLookExp in the repo. Returning that.") return exp except LookupError: pass try: raw = self.butler.get('raw', dataId=dataId) except LookupError: raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None # default options that are probably good for most engineering time isrConfig = IsrTask.ConfigClass() isrConfig.doWrite = False # this task writes separately, no need for this isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm isrConfig.doSaturationInterpolation = True isrConfig.overscanNumLeadingColumnsToSkip = 5 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW' # apply general overrides self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions) # apply per-image overrides self._applyConfigOverrides(isrConfig, extraIsrOptions) isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel', 'bfGains', 'ptc'] isrDict = {} # we build a cache of all the isr components which will be used to save # the IO time on subsequent calls. This assumes people will not update # calibration products while this object lives, but this is a fringe # use case, and if they do, all they would need to do would be call # .clearCache() and this will rebuild with the new products. for component in isrParts: if component in self._cache and component != 'flat': self.log.info(f"Using {component} from cache...") isrDict[component] = self._cache[component] continue try: # TODO: add caching for flats item = self.butler.get(component, dataId=dataId) self._cache[component] = item isrDict[component] = self._cache[component] except (RuntimeError, LookupError, OperationalError): pass quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure if self.doWrite: try: self.butler.put(quickLookExp, self._datasetName, dataId) self.log.info(f'Put {self._datasetName} for {dataId}') except ConflictingDefinitionError: # TODO: DM-34302 fix this message so that it's less scary for # users. Do this by having daemons know they're daemons. self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)') pass return quickLookExp
def run(self, ccdExposure, *, camera=None, bias=None, dark=None, flat=None, defects=None, linearizer=None, crosstalk=None, bfKernel=None, bfGains=None, ptc=None, crosstalkSources=None, isrBaseConfig=None): """Run isr and cosmic ray repair using, doing as much isr as possible. Retrieves as many calibration products as are available, and runs isr with those settings enabled, but always returns an assembled image at a minimum. Then performs cosmic ray repair if configured to. Parameters ---------- ccdExposure : `lsst.afw.image.Exposure` The raw exposure that is to be run through ISR. The exposure is modified by this method. camera : `lsst.afw.cameraGeom.Camera`, optional The camera geometry for this exposure. Required if one or more of ``ccdExposure``, ``bias``, ``dark``, or ``flat`` does not have an associated detector. bias : `lsst.afw.image.Exposure`, optional Bias calibration frame. linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional Functor for linearization. crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional Calibration for crosstalk. crosstalkSources : `list`, optional List of possible crosstalk sources. dark : `lsst.afw.image.Exposure`, optional Dark calibration frame. flat : `lsst.afw.image.Exposure`, optional Flat calibration frame. ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional Photon transfer curve dataset, with, e.g., gains and read noise. bfKernel : `numpy.ndarray`, optional Brighter-fatter kernel. bfGains : `dict` of `float`, optional Gains used to override the detector's nominal gains for the brighter-fatter correction. A dict keyed by amplifier name for the detector in question. defects : `lsst.ip.isr.Defects`, optional List of defects. fringes : `lsst.pipe.base.Struct`, optional Struct containing the fringe correction data, with elements: - ``fringes``: fringe calibration frame (`afw.image.Exposure`) - ``seed``: random seed derived from the ccdExposureId for random number generator (`uint32`) opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional A ``TransmissionCurve`` that represents the throughput of the, optics, to be evaluated in focal-plane coordinates. filterTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the filter itself, to be evaluated in focal-plane coordinates. sensorTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the sensor itself, to be evaluated in post-assembly trimmed detector coordinates. atmosphereTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the atmosphere, assumed to be spatially constant. detectorNum : `int`, optional The integer number for the detector to process. strayLightData : `object`, optional Opaque object containing calibration information for stray-light correction. If `None`, no correction will be performed. illumMaskedImage : `lsst.afw.image.MaskedImage`, optional Illumination correction image. isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional An isrTask config to act as the base configuration. Options which involve applying a calibration product are ignored, but this allows for the configuration of e.g. the number of overscan columns. Returns ------- result : `lsst.pipe.base.Struct` Result struct with component: - ``exposure`` : `afw.image.Exposure` The ISRed and cosmic-ray-repaired exposure. """ isrConfig = isrBaseConfig if isrBaseConfig else IsrTask.ConfigClass() isrConfig.doBias = False isrConfig.doDark = False isrConfig.doFlat = False isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doLinearize = False isrConfig.doCrosstalk = False isrConfig.doBrighterFatter = False isrConfig.usePtcGains = False if bias: isrConfig.doBias = True self.log.info("Running with bias correction") if dark: isrConfig.doDark = True self.log.info("Running with dark correction") if flat: isrConfig.doFlat = True self.log.info("Running with flat correction") # TODO: deal with fringes here if defects: isrConfig.doDefect = True self.log.info("Running with defect correction") if linearizer: isrConfig.doLinearize = True self.log.info("Running with linearity correction") if crosstalk: isrConfig.doCrosstalk = True self.log.info("Running with crosstalk correction") if bfKernel: isrConfig.doBrighterFatter = True self.log.info("Running with brighter-fatter correction") if ptc: isrConfig.usePtcGains = True self.log.info("Running with ptc correction") isrConfig.doWrite = False isrTask = IsrTask(config=isrConfig) result = isrTask.run( ccdExposure, camera=camera, bias=bias, dark=dark, flat=flat, # fringes=pipeBase.Struct(fringes=None), defects=defects, linearizer=linearizer, crosstalk=crosstalk, bfKernel=bfKernel, bfGains=bfGains, ptc=ptc, crosstalkSources=crosstalkSources, isGen3=True, ) postIsr = result.exposure if self.config.doRepairCosmics: try: # can fail due to too many CRs detected, and we always want an exposure back self.log.info("Repairing cosmics...") if postIsr.getPsf() is None: installPsfTask = InstallGaussianPsfTask() installPsfTask.run(postIsr) # TODO: try adding a reasonably wide Gaussian as a temp PSF # and then just running repairTask on its own without any # imChar. It should work, and be faster. repairConfig = CharacterizeImageTask.ConfigClass() repairConfig.doMeasurePsf = False repairConfig.doApCorr = False repairConfig.doDeblend = False repairConfig.doWrite = False repairConfig.repair.cosmicray.nCrPixelMax = 200000 repairTask = CharacterizeImageTask(config=repairConfig) repairTask.repair.run(postIsr) except Exception as e: self.log.warning(f"During CR repair caught: {e}") # exposure is returned for convenience to mimic isrTask's API. return pipeBase.Struct(exposure=postIsr, outputExposure=postIsr)