def testRunWithAddDistortionModel(self): """Test IsrTask.run with config.doAddDistortionModel true""" isrConfig = self.makeMinimalIsrConfig() isrConfig.doAddDistortionModel = True isrTask = IsrTask(config=isrConfig) with self.assertRaises(RuntimeError): # the camera argument is required isrTask.run(ccdExposure=self.exposure) exposure = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure desiredWcs = self.makeDesiredDistortedWcs() self.assertWcsAlmostEqualOverBBox(desiredWcs, exposure.getWcs(), self.bbox)
def testRunWithoutAddDistortionModel(self): """Test IsrTask.run with config.doAddDistortionModel false""" isrConfig = self.makeMinimalIsrConfig() isrTask = IsrTask(config=isrConfig) # the camera argument is not needed exposure = isrTask.run(ccdExposure=self.exposure).exposure self.assertEqual(self.wcs, exposure.getWcs()) # and the camera argument is ignored if provided exposure2 = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure self.assertEqual(self.wcs, exposure2.getWcs())
def testRunWithoutAddDistortionModel(self): """Test IsrTask.run with config.doAddDistortionModel false""" isrConfig = self.makeMinimalIsrConfig() isrTask = IsrTask(config=isrConfig) # the camera argument is not needed exposure = isrTask.run(ccdExposure=self.exposure).exposure self.assertEqual(self.wcs, exposure.getWcs()) # and the camera argument is ignored if provided exposure2 = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure self.assertEqual(self.wcs, exposure2.getWcs())
def testRunWithAddDistortionModel(self): """Test IsrTask.run with config.doAddDistortionModel true""" isrConfig = self.makeMinimalIsrConfig() isrConfig.doAddDistortionModel = True isrTask = IsrTask(config=isrConfig) with self.assertRaises(RuntimeError): # the camera argument is required isrTask.run(ccdExposure=self.exposure) exposure = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure desiredWcs = self.makeDesiredDistortedWcs() self.assertWcsAlmostEqualOverBBox(desiredWcs, exposure.getWcs(), self.bbox)
def execute(self, dataRef): """!Apply common instrument signature correction algorithms to a raw frame @param dataRef: butler data reference @return a pipeBase Struct containing: - exposure similar to IsrTask.runDataRef() """ self.log.info("Performing Super ISR on sensor data ID %s" % (dataRef.dataId, )) # IsrTask.runDataRef includes these three steps self.log.info("Reading input data using dataRef") inputData = self.read_input_data(dataRef) self.log.info( "Running operations. The run() method should not take anything Butler" ) result = IsrTask.run(IsrTask(self.config), **inputData.getDict()) self.log.info("Writing output data using dataRef") self.write_output_data(dataRef, result) return result
def runIsr(): '''Run the task to do ISR on a ccd''' #Create the isr task with modified config isrConfig = IsrTask.ConfigClass() isrConfig.doBias = False #We didn't make a zero frame isrConfig.doDark = True isrConfig.doFlat = True isrConfig.doFringe = False #There is no fringe frame for this example isrConfig.assembleCcd.setGain = False isrTask = IsrTask(config=isrConfig) #Make raw, flat and dark exposures DARKVAL = 2. #e-/sec OSCAN = 1000. #DN GRADIENT = .10 EXPTIME = 15 #seconds DARKEXPTIME = 40. #seconds darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME) flatExposure = exampleUtils.makeFlat(GRADIENT) rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME) output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure) return output.exposure
def runIsr(): '''Run the task to do ISR on a ccd''' #Create the isr task with modified config isrConfig = IsrTask.ConfigClass() isrConfig.doBias = False #We didn't make a zero frame isrConfig.doDark = True isrConfig.doFlat = True isrConfig.doFringe = False #There is no fringe frame for this example isrConfig.assembleCcd.doRenorm = False #We'll take care of gain in the flats isrConfig.assembleCcd.setGain = False isrTask = IsrTask(config=isrConfig) #Make raw, flat and dark exposures DARKVAL = 2. #e-/sec OSCAN = 1000. #DN GRADIENT = .10 EXPTIME = 15 #seconds DARKEXPTIME = 40. #seconds darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME) flatExposure = exampleUtils.makeFlat(GRADIENT) rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME) output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure) return output.exposure
def processExposure(exposure, bias=None, defects=None, repair=False, repo=None): from lsst.ip.isr import IsrTask import lsst.daf.persistence as dafPersist import time config = IsrTask.ConfigClass() if (bias == None or defects == None) and repo == None: raise (AttributeError( 'Repo keyword not set and is required to find bias/defect frames')) else: butler = dafPersist.Butler(repo) if False: config.overscanFitType = "AKIMA_SPLINE" config.overscanOrder = 5 else: config.overscanFitType = "POLY" config.overscanOrder = 3 config.doBias = True if config.doBias == True and bias == None: bias = butler.get( 'bias', visit=exposure.getInfo().getVisitInfo().getExposureId(), ) config.doDark = False config.doFringe = False config.doFlat = False config.doLinearize = False # TODO: should be able to run this but there are issues with defects in the stack at the moment config.doDefect = True if repair == True else False if (config.doDefect == True and bias == None): butler = dafPersist.Butler(repo) defects = butler.get( 'defects', visit=exposure.getInfo().getVisitInfo().getExposureId(), ) config.doAddDistortionModel = False config.doSaturationInterpolation = False config.overscanNumLeadingColumnsToSkip = 20 isrTask = IsrTask(config=config) # Run ISR start = time.time() isr_corrected_exposure = isrTask.run(exposure, bias=bias, defects=defects).exposure end = time.time() print('Time to perform image ISR was {0:2f} [s]'.format(end - start)) return (isr_corrected_exposure)
def run(self, ccdExposure, bias=None, dark=None, flat=None, defects=None, fringes=None): """Perform instrument signature removal on an exposure Steps include: - Detect saturation, apply overscan correction, bias, dark and flat - Perform CCD assembly - Interpolate over defects, saturated pixels and all NaNs - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True @param[in] ccdExposure -- lsst.afw.image.exposure of detector data @param[in] bias -- exposure of bias frame @param[in] dark -- exposure of dark frame @param[in] flat -- exposure of flatfield @param[in] defects -- list of detects @param[in] fringes -- exposure of fringe frame or list of fringe exposure @return a pipeBase.Struct with fields: - exposure: the exposure after application of ISR """ ccd = ccdExposure.getDetector() floatExposure = self.convertIntToFloat(ccdExposure) metadata = floatExposure.getMetadata() # Detect saturation # Saturation values recorded in the fits hader is not reliable, try to estimate it from the pixel vales # Find the peak location in the high end part the pixel values' histogram and set the saturation level at # safe * (peak location) where safe is a configurable parameter (typically 0.95) image = floatExposure.getMaskedImage().getImage() imageArray = image.getArray() maxValue = np.max(imageArray) if maxValue > 60000.0: hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue + 1.0)) saturate = int(self.config.safe * bin_edges[np.argmax(hist)]) else: saturate = metadata.get("SATURATE") self.log.info("Saturation set to %d" % saturate) for amp in ccd: amp.setSaturation(saturate) if amp.getName() == "A": amp.setGain(metadata.get("GAINA")) amp.setReadNoise(metadata.get("RDNOISEA")) elif amp.getName() == "B": amp.setGain(metadata.get("GAINB")) amp.setReadNoise(metadata.get("RDNOISEB")) else: raise ValueError("Unexpected amplifier name : %s" % (amp.getName())) return IsrTask.run( self, ccdExposure=ccdExposure, bias=bias, dark=dark, flat=flat, defects=defects, fringes=fringes )
def run(self, ccdExposure, *, camera=None, bias=None, dark=None, flat=None, defects=None, linearizer=None, crosstalk=None, bfKernel=None, bfGains=None, ptc=None, crosstalkSources=None, isrBaseConfig=None): """Run isr and cosmic ray repair using, doing as much isr as possible. Retrieves as many calibration products as are available, and runs isr with those settings enabled, but always returns an assembled image at a minimum. Then performs cosmic ray repair if configured to. Parameters ---------- ccdExposure : `lsst.afw.image.Exposure` The raw exposure that is to be run through ISR. The exposure is modified by this method. camera : `lsst.afw.cameraGeom.Camera`, optional The camera geometry for this exposure. Required if one or more of ``ccdExposure``, ``bias``, ``dark``, or ``flat`` does not have an associated detector. bias : `lsst.afw.image.Exposure`, optional Bias calibration frame. linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional Functor for linearization. crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional Calibration for crosstalk. crosstalkSources : `list`, optional List of possible crosstalk sources. dark : `lsst.afw.image.Exposure`, optional Dark calibration frame. flat : `lsst.afw.image.Exposure`, optional Flat calibration frame. ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional Photon transfer curve dataset, with, e.g., gains and read noise. bfKernel : `numpy.ndarray`, optional Brighter-fatter kernel. bfGains : `dict` of `float`, optional Gains used to override the detector's nominal gains for the brighter-fatter correction. A dict keyed by amplifier name for the detector in question. defects : `lsst.ip.isr.Defects`, optional List of defects. fringes : `lsst.pipe.base.Struct`, optional Struct containing the fringe correction data, with elements: - ``fringes``: fringe calibration frame (`afw.image.Exposure`) - ``seed``: random seed derived from the ccdExposureId for random number generator (`uint32`) opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional A ``TransmissionCurve`` that represents the throughput of the, optics, to be evaluated in focal-plane coordinates. filterTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the filter itself, to be evaluated in focal-plane coordinates. sensorTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the sensor itself, to be evaluated in post-assembly trimmed detector coordinates. atmosphereTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the atmosphere, assumed to be spatially constant. detectorNum : `int`, optional The integer number for the detector to process. strayLightData : `object`, optional Opaque object containing calibration information for stray-light correction. If `None`, no correction will be performed. illumMaskedImage : `lsst.afw.image.MaskedImage`, optional Illumination correction image. isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional An isrTask config to act as the base configuration. Options which involve applying a calibration product are ignored, but this allows for the configuration of e.g. the number of overscan columns. Returns ------- result : `lsst.pipe.base.Struct` Result struct with component: - ``exposure`` : `afw.image.Exposure` The ISRed and cosmic-ray-repaired exposure. """ isrConfig = isrBaseConfig if isrBaseConfig else IsrTask.ConfigClass() isrConfig.doBias = False isrConfig.doDark = False isrConfig.doFlat = False isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doLinearize = False isrConfig.doCrosstalk = False isrConfig.doBrighterFatter = False isrConfig.usePtcGains = False if bias: isrConfig.doBias = True self.log.info("Running with bias correction") if dark: isrConfig.doDark = True self.log.info("Running with dark correction") if flat: isrConfig.doFlat = True self.log.info("Running with flat correction") # TODO: deal with fringes here if defects: isrConfig.doDefect = True self.log.info("Running with defect correction") if linearizer: isrConfig.doLinearize = True self.log.info("Running with linearity correction") if crosstalk: isrConfig.doCrosstalk = True self.log.info("Running with crosstalk correction") if bfKernel: isrConfig.doBrighterFatter = True self.log.info("Running with brighter-fatter correction") if ptc: isrConfig.usePtcGains = True self.log.info("Running with ptc correction") isrConfig.doWrite = False isrTask = IsrTask(config=isrConfig) result = isrTask.run( ccdExposure, camera=camera, bias=bias, dark=dark, flat=flat, # fringes=pipeBase.Struct(fringes=None), defects=defects, linearizer=linearizer, crosstalk=crosstalk, bfKernel=bfKernel, bfGains=bfGains, ptc=ptc, crosstalkSources=crosstalkSources, isGen3=True, ) postIsr = result.exposure if self.config.doRepairCosmics: try: # can fail due to too many CRs detected, and we always want an exposure back self.log.info("Repairing cosmics...") if postIsr.getPsf() is None: installPsfTask = InstallGaussianPsfTask() installPsfTask.run(postIsr) # TODO: try adding a reasonably wide Gaussian as a temp PSF # and then just running repairTask on its own without any # imChar. It should work, and be faster. repairConfig = CharacterizeImageTask.ConfigClass() repairConfig.doMeasurePsf = False repairConfig.doApCorr = False repairConfig.doDeblend = False repairConfig.doWrite = False repairConfig.repair.cosmicray.nCrPixelMax = 200000 repairTask = CharacterizeImageTask(config=repairConfig) repairTask.repair.run(postIsr) except Exception as e: self.log.warning(f"During CR repair caught: {e}") # exposure is returned for convenience to mimic isrTask's API. return pipeBase.Struct(exposure=postIsr, outputExposure=postIsr)
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, displayResults=[], verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() isrTask = IsrTask(config=config.isr) calibrateTask = CalibrateTask(config=config.calibrate) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement, schema=schema, algMetadata=algMetadata) sourceMeasurementTask.config.doApplyApCorr = 'yes' # # Add fields needed to identify stars while calibrating # keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected", "Source was detected by calibrate")), None)] for key in calibrateTask.getCalibKeys(): keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key)) exposureDict = {}; calibSourcesDict = {}; sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance) # if config.interpPlanes: import lsst.ip.isr as ipIsr defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes, growFootprints=0) isrTask.run(exposure, defects=defects) # # process the data # if config.doCalibrate: result = calibrateTask.run(exposure) exposure, calibSources = result.exposure, result.sources else: calibSources = None if not exposure.getPsf(): calibrateTask.installInitialPsf(exposure) exposureDict[inputFile] = exposure calibSourcesDict[inputFile] = calibSources if returnCalibSources else None result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources, exposure.getPsf()) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) propagateCalibFlags(keysToCopy, calibSources, sources) if displayResults: # display results of processing (see also --debug argparse option) showApertures = "showApertures".upper() in displayResults showShapes = "showShapes".upper() in displayResults display = afwDisplay.getDisplay(frame=1) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = [] display.mtv(exposure, title=os.path.split(inputFile)[1]) with display.Buffering(): for s in sources: xy = s.getCentroid() display.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) if showShapes: display.dot(s.getShape(), *xy, ctype=afwDisplay.RED) if showApertures: for radius in radii: display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW) return exposureDict, calibSourcesDict, sourcesDict
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2") # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2") calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] isrResult1 = IsrTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get( "postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) icResult1 = CharacterizeImageTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure, exposureIdInfo=exposureIdInfo) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get( "icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background) calResult1 = CalibrateTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat, exposureIdInfo=exposureIdInfo, ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get( "calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual(calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent")) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, displayResults=[], verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() isrTask = IsrTask(config=config.isr) charImageTask = CharacterizeImageTask(None, config=config.charImage) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask( schema=schema, config=config.measurement, algMetadata=algMetadata) keysToCopy = [] for key in [ charImageTask.measurePsf.reservedKey, charImageTask.measurePsf.usedKey, ]: keysToCopy.append( (schema.addField(charImageTask.schema.find(key).field), key)) exposureDict = {} calibSourcesDict = {} sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance) # if config.interpPlanes: import lsst.ip.isr as ipIsr defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes, growFootprints=0) isrTask.run(exposure, defects=defects) # # process the data # if config.doCalibrate: result = charImageTask.characterize(exposure) exposure, calibSources = result.exposure, result.sourceCat else: calibSources = None if not exposure.getPsf(): charImageTask.installSimplePsf.run(exposure) exposureDict[inputFile] = exposure calibSourcesDict[ inputFile] = calibSources if returnCalibSources else None result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) propagatePsfFlags(keysToCopy, calibSources, sources) if displayResults: # display results of processing (see also --debug argparse option) showApertures = "showApertures".upper() in displayResults showPSFs = "showPSFs".upper() in displayResults showShapes = "showShapes".upper() in displayResults display = afwDisplay.getDisplay(frame=1) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = [] display.mtv(exposure, title=os.path.split(inputFile)[1]) with display.Buffering(): for s in sources: xy = s.getCentroid() display.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) if showPSFs and (s.get("calib_psfUsed") or s.get("calib_psfReserved")): display.dot( 'o', *xy, size=10, ctype=afwDisplay.GREEN if s.get("calib_psfUsed") else afwDisplay.YELLOW) if showShapes: display.dot(s.getShape(), *xy, ctype=afwDisplay.RED) if showApertures: for radius in radii: display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW) return exposureDict, calibSourcesDict, sourcesDict
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, **kwds): """Perform instrument signature removal on an exposure Steps include: - Detect saturation, apply overscan correction, bias, dark and flat - Perform CCD assembly - Interpolate over defects, saturated pixels and all NaNs - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True Parameters ---------- ccdExposure : `lsst.afw.image.Exposure` Detector data. bias : `lsst.afw.image.exposure` Exposure of bias frame. linearizer : `lsst.ip.isr.LinearizeBase` callable Linearizing functor; a subclass of lsst.ip.isr.LinearizeBase. dark : `lsst.afw.image.exposure` Exposure of dark frame. flat : `lsst.afw.image.exposure` Exposure of flatfield. defects : `list` list of detects fringes : `lsst.afw.image.Exposure` or list `lsst.afw.image.Exposure` exposure of fringe frame or list of fringe exposure bfKernel : None kernel used for brighter-fatter correction; currently unsupported **kwds : `dict` additional kwargs forwarded to IsrTask.run. Returns ------- struct : `lsst.pipe.base.Struct` with fields: - exposure: the exposure after application of ISR """ if bfKernel is not None: raise ValueError( "CFHT ISR does not currently support brighter-fatter correction." ) ccd = ccdExposure.getDetector() floatExposure = self.convertIntToFloat(ccdExposure) metadata = floatExposure.getMetadata() # Detect saturation # Saturation values recorded in the fits hader is not reliable, try to # estimate it from the pixel values. # Find the peak location in the high end part the pixel values' # histogram and set the saturation level at safe * (peak location) # where safe is a configurable parameter (typically 0.95) image = floatExposure.getMaskedImage().getImage() imageArray = image.getArray() maxValue = np.max(imageArray) if maxValue > 60000.0: hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue + 1.0)) saturate = int(self.config.safe * bin_edges[np.argmax(hist)]) else: saturate = metadata.getScalar("SATURATE") self.log.info("Saturation set to %d" % saturate) tempCcd = ccd.rebuild() tempCcd.clear() for amp in ccd: tempAmp = amp.rebuild() tempAmp.setSaturation(saturate) if tempAmp.getName() == "A": tempAmp.setGain(metadata.getScalar("GAINA")) rdnA = metadata.getScalar("RDNOISEA") # Check if the noise value is making sense for this amp. If # not, replace with value stored in RDNOISE slot. This change # is necessary to process some old CFHT images # (visit : 7xxxxx) where RDNOISEA/B = 65535 if rdnA > 60000.0: rdnA = metadata.getScalar("RDNOISE") tempAmp.setReadNoise(rdnA) elif tempAmp.getName() == "B": tempAmp.setGain(metadata.getScalar("GAINB")) rdnB = metadata.getScalar("RDNOISEB") # Check if the noise value is making sense for this amp. # If not, replace with value # stored in RDNOISE slot. This change is necessary to process # some old CFHT images # (visit : 7xxxxx) where RDNOISEA/B = 65535 if rdnB > 60000.0: rdnB = metadata.getScalar("RDNOISE") tempAmp.setReadNoise(rdnB) else: raise ValueError("Unexpected amplifier name : %s" % (amp.getName())) tempCcd.append(tempAmp) ccd = tempCcd.finish() ccdExposure.setDetector(ccd) return IsrTask.run(self, ccdExposure=ccdExposure, bias=bias, linearizer=linearizer, dark=dark, flat=flat, defects=defects, fringes=fringes, **kwds)
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None): """Perform instrument signature removal on an exposure Steps include: - Detect saturation, apply overscan correction, bias, dark and flat - Perform CCD assembly - Interpolate over defects, saturated pixels and all NaNs - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True @param[in] ccdExposure -- lsst.afw.image.exposure of detector data @param[in] bias -- exposure of bias frame @param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isr.LinearizeBase @param[in] dark -- exposure of dark frame @param[in] flat -- exposure of flatfield @param[in] defects -- list of detects @param[in] fringes -- exposure of fringe frame or list of fringe exposure @param[in] bfKernel - kernel used for brighter-fatter correction; currently unsupported @return a pipeBase.Struct with fields: - exposure: the exposure after application of ISR """ if bfKernel is not None: raise ValueError( "CFHT ISR does not currently support brighter-fatter correction." ) ccd = ccdExposure.getDetector() floatExposure = self.convertIntToFloat(ccdExposure) metadata = floatExposure.getMetadata() # Detect saturation # Saturation values recorded in the fits hader is not reliable, try to estimate it from # the pixel vales # Find the peak location in the high end part the pixel values' histogram and set the saturation # level at safe * (peak location) where safe is a configurable parameter (typically 0.95) image = floatExposure.getMaskedImage().getImage() imageArray = image.getArray() maxValue = np.max(imageArray) if maxValue > 60000.0: hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue + 1.0)) saturate = int(self.config.safe * bin_edges[np.argmax(hist)]) else: saturate = metadata.get("SATURATE") self.log.info("Saturation set to %d" % saturate) for amp in ccd: amp.setSaturation(saturate) if amp.getName() == "A": amp.setGain(metadata.get("GAINA")) rdnA = metadata.get("RDNOISEA") # Check if the noise value is making sense for this amp. If not, replace with value # stored in RDNOISE slot. This change is necessary to process some old CFHT images # (visit : 7xxxxx) where RDNOISEA/B = 65535 if rdnA > 60000.0: rdnA = metadata.get("RDNOISE") amp.setReadNoise(rdnA) elif amp.getName() == "B": amp.setGain(metadata.get("GAINB")) rdnB = metadata.get("RDNOISEB") # Check if the noise value is making sense for this amp. If not, replace with value # stored in RDNOISE slot. This change is necessary to process some old CFHT images # (visit : 7xxxxx) where RDNOISEA/B = 65535 if rdnB > 60000.0: rdnB = metadata.get("RDNOISE") amp.setReadNoise(rdnB) else: raise ValueError("Unexpected amplifier name : %s" % (amp.getName())) return IsrTask.run( self, ccdExposure=ccdExposure, bias=bias, linearizer=linearizer, dark=dark, flat=flat, defects=defects, fringes=fringes, )
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask( config=getObsTestConfig(IsrTask), name="isr2" ) # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2" ) calibrateTask = CalibrateTask( config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema ) try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()] isrResult1 = IsrTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) icResult1 = CharacterizeImageTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background ) calResult1 = CalibrateTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") ) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background ) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None, camera=None, **kwds): """Perform instrument signature removal on an exposure Steps include: - Detect saturation, apply overscan correction, bias, dark and flat - Perform CCD assembly - Interpolate over defects, saturated pixels and all NaNs - Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True Parameters ---------- ccdExposure : `lsst.afw.image.Exposure` Detector data. bias : `lsst.afw.image.exposure` Exposure of bias frame. linearizer : `lsst.ip.isr.LinearizeBase` callable Linearizing functor; a subclass of lsst.ip.isr.LinearizeBase. dark : `lsst.afw.image.exposure` Exposure of dark frame. flat : `lsst.afw.image.exposure` Exposure of flatfield. defects : `list` list of detects fringes : `lsst.afw.image.exposure` or `list` of `lsst.afw.image.exposure` exposure of fringe frame or list of fringe exposure bfKernel : None kernel used for brighter-fatter correction; currently unsupported camera : `lsst.afw.cameraGeom.Camera` Camera geometry, used by addDistortionModel. **kwds : `dict` additional kwargs forwarded to IsrTask.run. Returns ------- struct : `lsst.pipe.base.Struct` with fields: - exposure: the exposure after application of ISR """ if bfKernel is not None: raise ValueError("CFHT ISR does not currently support brighter-fatter correction.") ccd = ccdExposure.getDetector() floatExposure = self.convertIntToFloat(ccdExposure) metadata = floatExposure.getMetadata() # Detect saturation # Saturation values recorded in the fits hader is not reliable, try to # estimate it from the pixel values. # Find the peak location in the high end part the pixel values' # histogram and set the saturation level at safe * (peak location) # where safe is a configurable parameter (typically 0.95) image = floatExposure.getMaskedImage().getImage() imageArray = image.getArray() maxValue = np.max(imageArray) if maxValue > 60000.0: hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue+1.0)) saturate = int(self.config.safe*bin_edges[np.argmax(hist)]) else: saturate = metadata.getScalar("SATURATE") self.log.info("Saturation set to %d" % saturate) for amp in ccd: amp.setSaturation(saturate) if amp.getName() == "A": amp.setGain(metadata.getScalar("GAINA")) rdnA = metadata.getScalar("RDNOISEA") # Check if the noise value is making sense for this amp. If # not, replace with value stored in RDNOISE slot. This change # is necessary to process some old CFHT images # (visit : 7xxxxx) where RDNOISEA/B = 65535 if rdnA > 60000.0: rdnA = metadata.getScalar("RDNOISE") amp.setReadNoise(rdnA) elif amp.getName() == "B": amp.setGain(metadata.getScalar("GAINB")) rdnB = metadata.getScalar("RDNOISEB") # Check if the noise value is making sense for this amp. # If not, replace with value # stored in RDNOISE slot. This change is necessary to process # some old CFHT images # (visit : 7xxxxx) where RDNOISEA/B = 65535 if rdnB > 60000.0: rdnB = metadata.getScalar("RDNOISE") amp.setReadNoise(rdnB) else: raise ValueError("Unexpected amplifier name : %s"%(amp.getName())) return IsrTask.run(self, ccdExposure=ccdExposure, bias=bias, linearizer=linearizer, dark=dark, flat=flat, defects=defects, fringes=fringes, camera=camera, **kwds )