def testStretch(self): """Test playing with the lookup table""" ds9.show() ds9.scale("linear", "zscale")
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """!Determine a PCA PSF model for an exposure given a list of PSF candidates \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure) \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector \param[in,out] metadata a home for interesting tidbits of information \param[in] flagKey schema key used to mark sources actually used in PSF determination \return a list of - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info( __name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info( __name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info( __name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info( __name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info( __name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) # match Kernel amplitudes for spatial plots matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes # Keep matplotlib alive post mortem keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots displayPsfSpatialModel = lsstDebug.Info( __name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info( __name__).showBadCandidates # Include bad candidates # Normalize residuals by object amplitude normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals pause = lsstDebug.Info( __name__).pause # Prompt user after each iteration? if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.log.warn( "WARNING: NOT scaling kernelSize by stellar quadrupole moment " + "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead", self.config.kernelSize) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2 * int(self.config.kernelSize * math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print("Median size=%s" % (medSize, )) self.log.trace("Kernel size=%s", actualKernelSize) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [ ] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print("Removing %d blended Psf candidates" % len(blendedCandidates)) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW, size=4, frame=frame) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iterNum in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates # import lsst.afw.display.utils as displayUtils stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates ): # maybe include bad candidates try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append( (im, "%d%s" % (maUtils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception as e: continue if len(stamps) == 0: print( "WARNING: No PSF candidates to show; try setting showBadCandidates=True" ) else: mos = displayUtils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics( im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append( im, label, ds9.GREEN if status == afwMath.SpatialCellCandidate.GOOD else ds9.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED) mos.makeMosaic(frame=8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN ) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.log.debug("chi^2=%s; id=%s", cand.getChi2(), cand.getSource().getId()) for cand in awfulCandidates: if display: print("Removing bad candidate: id=%d, chi^2=%f" % \ (cand.getSource().getId(), cand.getChi2())) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates rchi2 = cand.getChi2( ) # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = psf.getKernel() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception as e: continue fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * k.getSum() predict = [ kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters()) ] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append( [a / amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:, k].mean() rms = residuals[:, k].std() elif False: # Using interquartile range sr = numpy.sort(residuals[:, k]) mean = sr[int(0.5*len(sr))] if len(sr) % 2 else \ 0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1]) rms = 0.74 * (sr[int(0.75 * len(sr))] - sr[int(0.25 * len(sr))]) else: stats = afwMath.makeStatistics( residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP) mean = stats.getValue(afwMath.MEANCLIP) rms = stats.getValue(afwMath.STDEVCLIP) rms = max( 1.0e-4, rms) # Don't trust RMS below this due to numerical issues if display: print("Mean for component %d is %f" % (k, mean)) print("RMS for component %d is %f" % (k, rms)) badCandidates = list() for i, cand in enumerate(candidates): if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject * rms: badCandidates.append(i) badCandidates.sort( key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates): cand = candidates[c] if display: print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \ (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k, residuals[badCandidates[i], k], self.config.spatialReject * rms)) cand.setStatus(afwMath.SpatialCellCandidate.BAD) # # Display results # if display and displayIterations: if displayExposure: if iterNum > 0: ds9.erase(frame=frame) maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", size=8, frame=frame, ctype=ds9.YELLOW, ctypeBad=ds9.RED, ctypeUnused=ds9.MAGENTA) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: maUtils.showPsfSpatialCells( exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", size=10, frame=frame, ctype=ds9.YELLOW, ctypeBad=ds9.RED) if displayResiduals: while True: try: maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = input( "Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:] else: reply = "" if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"): if reply == "c": pause = False elif reply == "h": print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \ "s[ave fileName] y[es]") continue elif reply == "p": import pdb pdb.set_trace() elif reply == "q": display = False elif reply == "Q": sys.exit(1) elif reply == "s": fileName = args.pop(0) if not fileName: print("Please provide a filename") continue print("Saving to %s" % fileName) maUtils.saveSpatialCellSet(psfCellSet, fileName=fileName) continue break else: print("Unrecognised response: %s" % reply, file=sys.stderr) if reply == "n": break # One last time, to take advantage of the last iteration psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # Display code for debugging # if display and reply != "n": if displayExposure: maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: maUtils.showPsfSpatialCells( exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=10, frame=frame) if displayResiduals: maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale("linear", 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) # # Generate some QA information # # Count PSF stars # numGoodStars = 0 numAvailStars = 0 avgX = 0.0 avgY = 0.0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # don't ignore BAD stars numAvailStars += 1 for cand in cell.begin(True): # do ignore BAD stars src = cand.getSource() if flagKey is not None: src.set(flagKey, True) avgX += src.getX() avgY += src.getY() numGoodStars += 1 avgX /= numGoodStars avgY /= numGoodStars if metadata is not None: metadata.set("spatialFitChi2", fitChi2) metadata.set("numGoodStars", numGoodStars) metadata.set("numAvailStars", numAvailStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psf = PcaPsf(psf.getKernel(), afwGeom.Point2D(avgX, avgY)) return psf, psfCellSet
action="store_true", help="Load debug.py?", default=False) parser.add_argument('--ds9', action="store_true", help="Display the result?", default=False) parser.add_argument('--write', '-w', action="store_true", help="Write the result?", default=False) args = parser.parse_args() if args.debug: try: import debug except ImportError as e: print(e, file=sys.stderr) exposure = runIsr() if args.ds9: im = exposure.getMaskedImage().getImage() im_median = numpy.median(im.getArray()) ds9.mtv(im) ds9.scale(min=im_median * 0.90, max=im_median * 1.1, type='SQRT') if args.write: exposure.writeFits("postISRCCD.fits")
flatExposure = exampleUtils.makeFlat(GRADIENT) rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME) output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure) return output.exposure if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="Demonstrate the use of IsrTask") parser.add_argument('--debug', '-d', action="store_true", help="Load debug.py?", default=False) parser.add_argument('--ds9', action="store_true", help="Display the result?", default=False) parser.add_argument('--write', '-w', action="store_true", help="Write the result?", default=False) args = parser.parse_args() if args.debug: try: import debug except ImportError as e: print >> sys.stderr, e exposure = runIsr() if args.ds9: im = exposure.getMaskedImage().getImage() im_median = numpy.median(im.getArray()) ds9.mtv(im) ds9.scale(min=im_median*0.90, max=im_median*1.1, type='SQRT') if args.write: exposure.writeFits("postISRCCD.fits")
showBadCandidates=showBadCandidates) maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = raw_input("Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:]
def run(self, exposure, sources, expId=0, matches=None): """!Measure the PSF \param[in,out] exposure Exposure to process; measured PSF will be added. \param[in,out] sources Measured sources on exposure; flag fields will be set marking stars chosen by the star selector and the PSF determiner if a schema was passed to the task constructor. \param[in] expId Exposure id used for generating random seed. \param[in] matches A list of lsst.afw.table.ReferenceMatch objects (\em i.e. of lsst.afw.table.Match with \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord --- the reference object and detected object respectively) as returned by \em e.g. the AstrometryTask. Used by star selectors that choose to refer to an external catalog. \return a pipe.base.Struct with fields: - psf: The measured PSF (also set in the input exposure) - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates as returned by the psf determiner. """ self.log.info("Measuring PSF") import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells displayPsfMosaic = lsstDebug.Info( __name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) displayPsfCandidates = lsstDebug.Info( __name__).displayPsfCandidates # show mosaic of candidates displayResiduals = lsstDebug.Info( __name__).displayResiduals # show residuals showBadCandidates = lsstDebug.Info( __name__).showBadCandidates # include bad candidates normalizeResiduals = lsstDebug.Info( __name__).normalizeResiduals # normalise residuals by object peak #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Run star selector # selectionResult = self.starSelector.run(exposure=exposure, sourceCat=sources, matches=matches) reserveResult = self.reserve.run(selectionResult.starCat, expId=expId) psfCandidateList = [ cand for cand, use in zip(selectionResult.psfCandidates, reserveResult.use) if use ] if psfCandidateList and self.candidateKey is not None: for cand in psfCandidateList: source = cand.getSource() source.set(self.candidateKey, True) self.log.info("PSF star selector found %d candidates" % len(psfCandidateList)) if display: frame = display if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Determine PSF # psf, cellSet = self.psfDeterminer.determinePsf(exposure, psfCandidateList, self.metadata, flagKey=self.usedKey) self.log.info("PSF determination using %d/%d stars." % (self.metadata.get("numGoodStars"), self.metadata.get("numAvailStars"))) exposure.setPsf(psf) if display: frame = display if displayExposure: showPsfSpatialCells(exposure, cellSet, showBadCandidates, frame=frame) frame += 1 if displayPsfCandidates: # Show a mosaic of PSF candidates plotPsfCandidates(cellSet, showBadCandidates, frame) frame += 1 if displayResiduals: frame = plotResiduals(exposure, cellSet, showBadCandidates=showBadCandidates, normalizeResiduals=normalizeResiduals, frame=frame) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=frame, showFwhm=True) ds9.scale(0, 1, "linear", frame=frame) frame += 1 return pipeBase.Struct( psf=psf, cellSet=cellSet, )
def run(self, exposure, sources, expId=0, matches=None): """!Measure the PSF \param[in,out] exposure Exposure to process; measured PSF will be added. \param[in,out] sources Measured sources on exposure; flag fields will be set marking stars chosen by the star selector and the PSF determiner if a schema was passed to the task constructor. \param[in] expId Exposure id used for generating random seed. \param[in] matches A list of lsst.afw.table.ReferenceMatch objects (\em i.e. of lsst.afw.table.Match with \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord --- the reference object and detected object respectively) as returned by \em e.g. the AstrometryTask. Used by star selectors that choose to refer to an external catalog. \return a pipe.base.Struct with fields: - psf: The measured PSF (also set in the input exposure) - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates as returned by the psf determiner. """ self.log.info("Measuring PSF") import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show mosaic of candidates displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # include bad candidates normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals # normalise residuals by object peak #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Run star selector # psfCandidateList = self.starSelector.run(exposure=exposure, sourceCat=sources, matches=matches).psfCandidates reserveList = [] if self.config.reserveFraction > 0 : random.seed(self.config.reserveSeed*expId) reserveList = random.sample(psfCandidateList, int((self.config.reserveFraction)*len(psfCandidateList))) for cand in reserveList: psfCandidateList.remove(cand) if reserveList and self.reservedKey is not None: for cand in reserveList: source = cand.getSource() source.set(self.reservedKey,True) if psfCandidateList and self.candidateKey is not None: for cand in psfCandidateList: source = cand.getSource() source.set(self.candidateKey, True) self.log.info("PSF star selector found %d candidates" % len(psfCandidateList)) if self.config.reserveFraction > 0 : self.log.info("Reserved %d candidates from the fitting" % len(reserveList)) if display: frame = display if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Determine PSF # psf, cellSet = self.psfDeterminer.determinePsf(exposure, psfCandidateList, self.metadata, flagKey=self.usedKey) self.log.info("PSF determination using %d/%d stars." % (self.metadata.get("numGoodStars"), self.metadata.get("numAvailStars"))) exposure.setPsf(psf) if display: frame = display if displayExposure: showPsfSpatialCells(exposure, cellSet, showBadCandidates, frame=frame) frame += 1 if displayPsfCandidates: # Show a mosaic of PSF candidates plotPsfCandidates(cellSet, showBadCandidates, frame) frame += 1 if displayResiduals: frame = plotResiduals(exposure, cellSet, showBadCandidates=showBadCandidates, normalizeResiduals=normalizeResiduals, frame=frame) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=frame, showFwhm=True) ds9.scale(0, 1, "linear", frame=frame) frame += 1 return pipeBase.Struct( psf = psf, cellSet = cellSet, )
def runDataRef(self, sensorRef): self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId)) ccdExposure = sensorRef.get('raw') if self.config.removePcCards: # Remove any PC00N00M cards in the header raw_md = sensorRef.get("raw_md") nPc = 0 for i in ( 1, 2, ): for j in ( 1, 2, ): k = "PC%03d%03d" % (i, j) for md in (raw_md, ccdExposure.getMetadata()): if md.exists(k): md.remove(k) nPc += 1 if nPc: self.log.info("Recreating Wcs after stripping PC00n00m" % (sensorRef.dataId)) ccdExposure.setWcs(afwImage.makeWcs(raw_md)) ccdExposure = self.convertIntToFloat(ccdExposure) ccd = ccdExposure.getDetector() # Read in defects to check for any dead amplifiers (entire amp is within defect region) defects = sensorRef.get("defects", immediate=True) for amp in ccd: # Check if entire amp region is defined as defect (need to use amp.getBBox() for correct # comparison with current defects definition. badAmp = bool( sum([v.getBBox().contains(amp.getBBox()) for v in defects])) # In the case of bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct # association with pixels in current ccdExposure). if badAmp: dataView = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawBBox(), afwImage.PARENT) maskView = dataView.getMask() maskView |= maskView.getPlaneBitMask("BAD") del maskView self.measureOverscan(ccdExposure, amp) if self.config.doSaturation and not badAmp: self.saturationDetection(ccdExposure, amp) if self.config.doSuspect and not badAmp: self.suspectDetection(ccdExposure, amp) ampMask = afwImage.Mask(ccdExposure.getMaskedImage().getMask(), amp.getRawDataBBox(), afwImage.PARENT) maskVal = ampMask.getPlaneBitMask( [self.config.saturatedMaskName, self.config.suspectMaskName]) if numpy.all(ampMask.getArray() & maskVal > 0): badAmp = True if self.config.doOverscan and not badAmp: ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(), afwImage.PARENT) overscan = afwImage.MaskedImageF( ccdExposure.getMaskedImage(), amp.getRawHorizontalOverscanBBox(), afwImage.PARENT) overscanArray = overscan.getImage().getArray() median = numpy.ma.median( numpy.ma.masked_where(overscan.getMask().getArray(), overscanArray)) bad = numpy.where( numpy.abs(overscanArray - median) > self.config.overscanMaxDev) overscan.getMask().getArray()[bad] = overscan.getMask( ).getPlaneBitMask("SAT") statControl = afwMath.StatisticsControl() statControl.setAndMask( ccdExposure.getMaskedImage().getMask().getPlaneBitMask( "SAT")) lsstIsr.overscanCorrection( ampMaskedImage=ampImage, overscanImage=overscan, fitType=self.config.overscanFitType, order=self.config.overscanOrder, collapseRej=self.config.overscanRej, statControl=statControl, ) ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) ccd = ccdExposure.getDetector() if self.config.qa.doWriteOss: sensorRef.put(ccdExposure, "ossImage") if self.config.qa.doThumbnailOss: self.writeThumbnail(sensorRef, "ossThumb", ccdExposure) if self.config.doBias: biasExposure = self.getIsrExposure(sensorRef, "bias") self.biasCorrection(ccdExposure, biasExposure) if self.config.doVariance: for amp in ccd: ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox(), afwImage.PARENT) self.updateVariance(ampExposure, amp) if self.doLinearize(ccd): # immediate=True required for functors and linearizers are functors; see ticket DM-6515 linearizer = sensorRef.get("linearizer", immediate=True) linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log) if self.config.doCrosstalk: self.crosstalk.run(ccdExposure) if self.config.doWidenSaturationTrails: self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) interpolationDone = False darkExposure = self.getIsrExposure( sensorRef, "dark") if self.config.doDark else None flatExposure = self.getIsrExposure( sensorRef, "flat") if self.config.doFlat else None if self.config.doBrighterFatter: # We need to apply flats and darks before we can interpolate, and we # need to interpolate before we do B-F, but we do B-F without the # flats and darks applied so we can work in units of electrons or holes. # This context manager applies and then removes the darks and flats. with self.flatContext(ccdExposure, flatExposure, darkExposure): if self.config.doDefect: self.maskAndInterpDefect(ccdExposure, defects) if self.config.doSaturationInterpolation: self.saturationInterpolation(ccdExposure) self.maskAndInterpNan(ccdExposure) interpolationDone = True brighterFatterKernel = sensorRef.get('bfKernel') self.brighterFatterCorrection( ccdExposure, brighterFatterKernel, self.config.brighterFatterMaxIter, self.config.brighterFatterThreshold, self.config.brighterFatterApplyGain, ) if self.config.doDark: self.darkCorrection(ccdExposure, darkExposure) if self.config.doStrayLight: self.strayLight.run(sensorRef, ccdExposure) if self.config.doFlat: self.flatCorrection(ccdExposure, flatExposure, doTweakFlat=self.config.doTweakFlat) if self.config.doDefect and not interpolationDone: self.maskAndInterpDefect(ccdExposure, defects) if self.config.doApplyGains: self.applyGains(ccdExposure, self.config.normalizeGains) if self.config.doSaturation and not interpolationDone: self.saturationInterpolation(ccdExposure) if self.config.doFringe: self.fringe.runDataRef(ccdExposure, sensorRef) if self.config.doSetBadRegions: self.setBadRegions(ccdExposure) if not interpolationDone or self.config.doNanInterpAfterFlat: self.maskAndInterpNan(ccdExposure) if self.config.qa.doWriteFlattened: sensorRef.put(ccdExposure, "flattenedImage") if self.config.qa.doThumbnailFlattened: self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure) self.measureBackground(ccdExposure) if self.config.doGuider: self.guider(ccdExposure) self.roughZeroPoint(ccdExposure) if self.config.doWriteVignettePolygon: self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) if self.config.doWrite: sensorRef.put(ccdExposure, "postISRCCD") if self._display: im = ccdExposure.getMaskedImage().getImage() im_median = float(numpy.median(im.getArray())) ds9.mtv(im) ds9.scale(min=im_median * 0.95, max=im_median * 1.15) return Struct(exposure=ccdExposure)
showBadCandidates=showBadCandidates, variance=True) except: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = raw_input( "Next iteration? [ynchpqQs] ").strip() except EOFError:
def runDataRef(self, sensorRef): self.log.log(self.log.INFO, "Performing ISR on sensor %s" % (sensorRef.dataId)) ccdExposure = sensorRef.get('raw') if self.config.removePcCards: # Remove any PC00N00M cards in the header raw_md = sensorRef.get("raw_md") nPc = 0 for i in ( 1, 2, ): for j in ( 1, 2, ): k = "PC%03d%03d" % (i, j) for md in (raw_md, ccdExposure.getMetadata()): if md.exists(k): md.remove(k) nPc += 1 if nPc: self.log.log( self.log.INFO, "Recreating Wcs after stripping PC00n00m" % (sensorRef.dataId)) ccdExposure.setWcs(afwImage.makeWcs(raw_md)) ccdExposure = self.convertIntToFloat(ccdExposure) ccd = ccdExposure.getDetector() for amp in ccd: self.measureOverscan(ccdExposure, amp) if self.config.doSaturation: self.saturationDetection(ccdExposure, amp) if self.config.doOverscan: ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(), afwImage.PARENT) overscan = afwImage.MaskedImageF( ccdExposure.getMaskedImage(), amp.getRawHorizontalOverscanBBox(), afwImage.PARENT) overscanArray = overscan.getImage().getArray() median = numpy.ma.median( numpy.ma.masked_where(overscan.getMask().getArray(), overscanArray)) bad = numpy.where( numpy.abs(overscanArray - median) > self.config.overscanMaxDev) overscan.getMask().getArray()[bad] = overscan.getMask( ).getPlaneBitMask("SAT") statControl = afwMath.StatisticsControl() statControl.setAndMask( ccdExposure.getMaskedImage().getMask().getPlaneBitMask( "SAT")) lsstIsr.overscanCorrection( ampMaskedImage=ampImage, overscanImage=overscan, fitType=self.config.overscanFitType, order=self.config.overscanOrder, collapseRej=self.config.overscanRej, statControl=statControl, ) if self.config.doVariance: # Ideally, this should be done after bias subtraction, # but CCD assembly demands a variance plane ampExposure = ccdExposure.Factory(ccdExposure, amp.getRawDataBBox(), afwImage.PARENT) self.updateVariance(ampExposure, amp) ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) ccd = ccdExposure.getDetector() doRotateCalib = False # Rotate calib images for bias/dark/flat correction? nQuarter = ccd.getOrientation().getNQuarter() if nQuarter != 0: doRotateCalib = True if self.config.doDefect: defects = sensorRef.get('defects', immediate=True) self.maskAndInterpDefect(ccdExposure, defects) if self.config.qa.doWriteOss: sensorRef.put(ccdExposure, "ossImage") if self.config.qa.doThumbnailOss: self.writeThumbnail(sensorRef, "ossThumb", ccdExposure) if self.config.doBias: biasExposure = self.getIsrExposure(sensorRef, "bias") if not doRotateCalib: self.biasCorrection(ccdExposure, biasExposure) else: with self.rotated(ccdExposure) as exp: self.biasCorrection(exp, biasExposure) if self.config.doLinearize: self.linearize(ccdExposure) if self.config.doCrosstalk: self.crosstalk.run(ccdExposure) if self.config.doDark: darkExposure = self.getIsrExposure(sensorRef, "dark") if not doRotateCalib: self.darkCorrection(ccdExposure, darkExposure) else: with self.rotated(ccdExposure) as exp: self.darkCorrection(exp, darkExposure) if self.config.doFlat: flatExposure = self.getIsrExposure(sensorRef, "flat") if not doRotateCalib: self.flatCorrection(ccdExposure, flatExposure) else: with self.rotated(ccdExposure) as exp: self.flatCorrection(exp, flatExposure) if self.config.doApplyGains: self.applyGains(ccdExposure, self.config.normalizeGains) if self.config.doWidenSaturationTrails: self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) if self.config.doSaturation: self.saturationInterpolation(ccdExposure) if self.config.doFringe: self.fringe.runDataRef(ccdExposure, sensorRef) if self.config.doSetBadRegions: self.setBadRegions(ccdExposure) self.maskAndInterpNan(ccdExposure) if self.config.qa.doWriteFlattened: sensorRef.put(ccdExposure, "flattenedImage") if self.config.qa.doThumbnailFlattened: self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure) self.measureBackground(ccdExposure) if self.config.doGuider: self.guider(ccdExposure) self.roughZeroPoint(ccdExposure) if self.config.doWriteVignettePolygon: self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) if self.config.doWrite: sensorRef.put(ccdExposure, "postISRCCD") if self._display: im = ccdExposure.getMaskedImage().getImage() im_median = float(numpy.median(im.getArray())) ds9.mtv(im) ds9.scale(min=im_median * 0.95, max=im_median * 1.15) return Struct(exposure=ccdExposure)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """Determine a PSFEX PSF model for an exposure given a list of PSF candidates @param[in] exposure: exposure containing the psf candidates (lsst.afw.image.Exposure) @param[in] psfCandidateList: a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector @param[in,out] metadata a home for interesting tidbits of information @param[in] flagKey: schema key used to mark sources actually used in PSF determination @return psf: a meas.extensions.psfex.PsfexPsf """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = display and \ lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfComponents = display and \ lsstDebug.Info(__name__).displayPsfComponents # show the basis functions showBadCandidates = display and \ lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks) displayResiduals = display and \ lsstDebug.Info(__name__).displayResiduals # show residuals displayPsfMosaic = display and \ lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals # Normalise residuals by object amplitude mi = exposure.getMaskedImage() nCand = len(psfCandidateList) if nCand == 0: raise RuntimeError("No PSF candidates supplied.") # # How big should our PSF models be? # if display: # only needed for debug plots # construct and populate a spatial cell set bbox = mi.getBBox(afwImage.PARENT) psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) else: psfCellSet = None sizes = np.empty(nCand) for i, psfCandidate in enumerate(psfCandidateList): try: if psfCellSet: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) rmsSize = quad.getTraceRadius() sizes[i] = rmsSize if self.config.kernelSize >= 15: self.log.warn( "NOT scaling kernelSize by stellar quadrupole moment, but using absolute value" ) actualKernelSize = int(self.config.kernelSize) else: actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: rms = np.median(sizes) print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2 * np.sqrt(2 * np.log(2)) * rms)) # If we manually set the resolution then we need the size in pixel units pixKernelSize = actualKernelSize if self.config.samplingSize > 0: pixKernelSize = int(actualKernelSize * self.config.samplingSize) if pixKernelSize % 2 == 0: pixKernelSize += 1 self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize) psfCandidateList[0].setHeight(pixKernelSize) psfCandidateList[0].setWidth(pixKernelSize) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX # # Insert the good candidates into the set # defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex") args_md = dafBase.PropertySet() args_md.set("BASIS_TYPE", str(self.config.psfexBasis)) args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder)) args_md.set("PSF_SIZE", str(actualKernelSize)) args_md.set("PSF_SAMPLING", str(self.config.samplingSize)) prefs = psfex.Prefs(defaultsFile, args_md) prefs.setCommandLine([]) prefs.addCatalog("psfexPsfDeterminer") prefs.use() principalComponentExclusionFlag = bool( bool(psfex.Context.REMOVEHIDDEN) if False else psfex.Context. KEEPHIDDEN) context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(), prefs.getGroupDeg(), principalComponentExclusionFlag) set = psfex.Set(context) set.setVigSize(pixKernelSize, pixKernelSize) set.setFwhm(2 * np.sqrt(2 * np.log(2)) * np.median(sizes)) set.setRecentroid(self.config.recentroid) catindex, ext = 0, 0 backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue() ccd = exposure.getDetector() if ccd: gain = np.mean(np.array([a.getGain() for a in ccd])) else: gain = 1.0 self.log.warn("Setting gain to %g" % (gain, )) contextvalp = [] for i, key in enumerate(context.getName()): if context.getPcflag(i): contextvalp.append(pcval[pc]) pc += 1 elif key[0] == ':': try: contextvalp.append(exposure.getMetadata().get(key[1:])) except KeyError: raise RuntimeError( "*Error*: %s parameter not found in the header of %s" % (key[1:], prefs.getContextName())) else: try: contextvalp.append( np.array([ psfCandidateList[_].getSource().get(key) for _ in range(nCand) ])) except KeyError: raise RuntimeError("*Error*: %s parameter not found" % (key, )) set.setContextname(i, key) if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits) fluxName = prefs.getPhotfluxRkey() fluxFlagName = "base_" + fluxName + "_flag" xpos, ypos = [], [] for i, psfCandidate in enumerate(psfCandidateList): source = psfCandidate.getSource() xc, yc = source.getX(), source.getY() try: int(xc), int(yc) except ValueError: continue try: pstamp = psfCandidate.getMaskedImage().clone() except Exception as e: continue if fluxFlagName in source.schema and source.get(fluxFlagName): continue flux = source.get(fluxName) if flux < 0 or np.isnan(flux): continue # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate). # Having created the sample, we must proceed to configure it, and then fini (finalize), # or it will be malformed. try: sample = set.newSample() sample.setCatindex(catindex) sample.setExtindex(ext) sample.setObjindex(i) imArray = pstamp.getImage().getArray() imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \ -2*psfex.BIG sample.setVig(imArray) sample.setNorm(flux) sample.setBacknoise2(backnoise2) sample.setGain(gain) sample.setX(xc) sample.setY(yc) sample.setFluxrad(sizes[i]) for j in range(set.getNcontext()): sample.setContext(j, float(contextvalp[j][i])) except Exception as e: self.log.debug( "Exception when processing sample at (%f,%f): %s", xc, yc, e) continue else: set.finiSample(sample) xpos.append(xc) # for QA ypos.append(yc) if displayExposure: with ds9.Buffering(): ds9.dot("o", xc, yc, ctype=ds9.CYAN, size=4, frame=frame) if set.getNsample() == 0: raise RuntimeError("No good PSF candidates to pass to PSFEx") #---- Update min and max and then the scaling for i in range(set.getNcontext()): cmin = contextvalp[i].min() cmax = contextvalp[i].max() set.setContextScale(i, cmax - cmin) set.setContextOffset(i, (cmin + cmax) / 2.0) # Don't waste memory! set.trimMemory() #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX # # Do a PSFEX decomposition of those PSF candidates # fields = [] field = psfex.Field("Unknown") field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample()) field.finalize() fields.append(field) sets = [] sets.append(set) psfex.makeit(fields, sets) psfs = field.getPsfs() # Flag which objects were actually used in psfex by good_indices = [] for i in range(sets[0].getNsample()): index = sets[0].getSample(i).getObjindex() if index > -1: good_indices.append(index) if flagKey is not None: for i, psfCandidate in enumerate(psfCandidateList): source = psfCandidate.getSource() if i in good_indices: source.set(flagKey, True) xpos = np.array(xpos) ypos = np.array(ypos) numGoodStars = len(good_indices) avgX, avgY = np.mean(xpos), np.mean(ypos) psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY)) if False and (displayResiduals or displayPsfMosaic): ext = 0 frame = 1 diagnostics = True catDir = "." title = "psfexPsfDeterminer" psfex.psfex.showPsf(psfs, set, ext, [(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())], nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir, title=title) # # Display code for debugging # if display: assert psfCellSet is not None if displayExposure: maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame) if displayResiduals: maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: maUtils.showPsf(psf, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) # # Generate some QA information # # Count PSF stars # if metadata is not None: metadata.set("spatialFitChi2", np.nan) metadata.set("numAvailStars", nCand) metadata.set("numGoodStars", numGoodStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psfCellSet = None return psf, psfCellSet
def runDataRef(self, sensorRef): self.log.log(self.log.INFO, "Performing ISR on sensor %s" % (sensorRef.dataId)) ccdExposure = sensorRef.get('raw') if self.config.removePcCards: # Remove any PC00N00M cards in the header raw_md = sensorRef.get("raw_md") nPc = 0 for i in (1, 2,): for j in (1, 2,): k = "PC%03d%03d" % (i, j) for md in (raw_md, ccdExposure.getMetadata()): if md.exists(k): md.remove(k) nPc += 1 if nPc: self.log.log(self.log.INFO, "Recreating Wcs after stripping PC00n00m" % (sensorRef.dataId)) ccdExposure.setWcs(afwImage.makeWcs(raw_md)) ccdExposure = self.convertIntToFloat(ccdExposure) ccd = ccdExposure.getDetector() # Read in defects to check for any dead amplifiers (entire amp is within defect region) defectsRaw = sensorRef.get("defects", immediate=True) # Need to rotate defects bbox if we are dealing with a rotated ccd as they are defined assuming # that (0, 0) is the lower-left corner (LLC). Rotation needs to be done in trimmed/assembled image # dimensions as that is the frame for which the defects are defined. This is computed here as # bboxTrimmed. Also need to accommodate even and odd number of 90deg rotations. bboxTrimmed = afwGeom.Box2I() for amp in ccd: bboxTrimmed.include(amp.getBBox()) nQuarter = ccd.getOrientation().getNQuarter() defects = [] if nQuarter != 0: for v in defectsRaw: rotBox = afwImage.imageLib.DefectBase( afwCG.rotateBBoxBy90(v.getBBox(), 4 - nQuarter, afwGeom.Extent2I(bboxTrimmed.getHeight(), bboxTrimmed.getHeight()))) # We have rotated about the center of a square with ccd height dimension on a side. # rotateBBoxBy90 rotates CCW, so using 4 - nQuarter we are effectively rotating # CW nQuarter turns. So, for nQuarter = 2 or 3, the rotated LLC will be shifted # in x by width - height pixels wrt the LLC of the rotation square. Thus defects # for all nQuarter > 1 ccds need to be shifted back in x by this amount. if nQuarter > 1: shiftBoxX0 = bboxTrimmed.getWidth() - bboxTrimmed.getHeight() shiftBoxY0 = 0 rotBox.shift(afwGeom.Extent2I(shiftBoxX0, shiftBoxY0)) defects.append(rotBox) else: defects = defectsRaw for amp in ccd: # Check if entire amp region is defined as defect (need to use amp.getBBox() for correct # comparison with current defects definition. badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects])) # In the case of bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct # association with pixels in current ccdExposure). if badAmp: dataView = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawBBox(), afwImage.PARENT) maskView = dataView.getMask() maskView |= maskView.getPlaneBitMask("BAD") del maskView self.measureOverscan(ccdExposure, amp) if self.config.doSaturation and not badAmp: self.saturationDetection(ccdExposure, amp) if self.config.doOverscan and not badAmp: ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(), afwImage.PARENT) overscan = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawHorizontalOverscanBBox(), afwImage.PARENT) overscanArray = overscan.getImage().getArray() median = numpy.ma.median(numpy.ma.masked_where(overscan.getMask().getArray(), overscanArray)) bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) overscan.getMask().getArray()[bad] = overscan.getMask().getPlaneBitMask("SAT") statControl = afwMath.StatisticsControl() statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT")) lsstIsr.overscanCorrection(ampMaskedImage=ampImage, overscanImage=overscan, fitType=self.config.overscanFitType, order=self.config.overscanOrder, collapseRej=self.config.overscanRej, statControl=statControl, ) if self.config.doVariance: # Ideally, this should be done after bias subtraction, # but CCD assembly demands a variance plane ampExposure = ccdExposure.Factory(ccdExposure, amp.getRawDataBBox(), afwImage.PARENT) self.updateVariance(ampExposure, amp) ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) ccd = ccdExposure.getDetector() doRotateCalib = False # Rotate calib images for bias/dark/flat correction? if nQuarter != 0: doRotateCalib = True if self.config.doDefect: self.maskAndInterpDefect(ccdExposure, defects) if self.config.qa.doWriteOss: sensorRef.put(ccdExposure, "ossImage") if self.config.qa.doThumbnailOss: self.writeThumbnail(sensorRef, "ossThumb", ccdExposure) if self.config.doBias: biasExposure = self.getIsrExposure(sensorRef, "bias") if not doRotateCalib: self.biasCorrection(ccdExposure, biasExposure) else: with self.rotated(ccdExposure) as exp: self.biasCorrection(exp, biasExposure) if self.config.doLinearize: self.linearize(ccdExposure) if self.config.doCrosstalk: self.crosstalk.run(ccdExposure) if self.config.doBrighterFatter: brighterFatterKernel = sensorRef.get('bfKernel') self.brighterFatterCorrection(ccdExposure, brighterFatterKernel, self.config.brighterFatterMaxIter, self.config.brighterFatterThreshold, self.config.brighterFatterApplyGain, ) if self.config.doDark: darkExposure = self.getIsrExposure(sensorRef, "dark") if not doRotateCalib: self.darkCorrection(ccdExposure, darkExposure) else: with self.rotated(ccdExposure) as exp: self.darkCorrection(exp, darkExposure) if self.config.doFlat: flatExposure = self.getIsrExposure(sensorRef, "flat") if not doRotateCalib: self.flatCorrection(ccdExposure, flatExposure) else: with self.rotated(ccdExposure) as exp: self.flatCorrection(exp, flatExposure) if self.config.doApplyGains: self.applyGains(ccdExposure, self.config.normalizeGains) if self.config.doWidenSaturationTrails: self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) if self.config.doSaturation: self.saturationInterpolation(ccdExposure) if self.config.doFringe: self.fringe.runDataRef(ccdExposure, sensorRef) if self.config.doSetBadRegions: self.setBadRegions(ccdExposure) self.maskAndInterpNan(ccdExposure) if self.config.qa.doWriteFlattened: sensorRef.put(ccdExposure, "flattenedImage") if self.config.qa.doThumbnailFlattened: self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure) self.measureBackground(ccdExposure) if self.config.doGuider: self.guider(ccdExposure) self.roughZeroPoint(ccdExposure) if self.config.doWriteVignettePolygon: self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) if self.config.doWrite: sensorRef.put(ccdExposure, "postISRCCD") if self._display: im = ccdExposure.getMaskedImage().getImage() im_median = float(numpy.median(im.getArray())) ds9.mtv(im) ds9.scale(min=im_median*0.95, max=im_median*1.15) return Struct(exposure=ccdExposure)
def runDataRef(self, sensorRef): self.log.log(self.log.INFO, "Performing ISR on sensor %s" % (sensorRef.dataId)) ccdExposure = sensorRef.get("raw") if self.config.removePcCards: # Remove any PC00N00M cards in the header raw_md = sensorRef.get("raw_md") nPc = 0 for i in (1, 2): for j in (1, 2): k = "PC%03d%03d" % (i, j) for md in (raw_md, ccdExposure.getMetadata()): if md.exists(k): md.remove(k) nPc += 1 if nPc: self.log.log(self.log.INFO, "Recreating Wcs after stripping PC00n00m" % (sensorRef.dataId)) ccdExposure.setWcs(afwImage.makeWcs(raw_md)) ccdExposure = self.convertIntToFloat(ccdExposure) ccd = ccdExposure.getDetector() for amp in ccd: self.measureOverscan(ccdExposure, amp) if self.config.doSaturation: self.saturationDetection(ccdExposure, amp) if self.config.doOverscan: ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(), afwImage.PARENT) overscan = afwImage.MaskedImageF( ccdExposure.getMaskedImage(), amp.getRawHorizontalOverscanBBox(), afwImage.PARENT ) overscanArray = overscan.getImage().getArray() median = numpy.ma.median(numpy.ma.masked_where(overscan.getMask().getArray(), overscanArray)) bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev) overscan.getMask().getArray()[bad] = overscan.getMask().getPlaneBitMask("SAT") statControl = afwMath.StatisticsControl() statControl.setAndMask(ccdExposure.getMaskedImage().getMask().getPlaneBitMask("SAT")) lsstIsr.overscanCorrection( ampMaskedImage=ampImage, overscanImage=overscan, fitType=self.config.overscanFitType, order=self.config.overscanOrder, collapseRej=self.config.overscanRej, statControl=statControl, ) if self.config.doVariance: # Ideally, this should be done after bias subtraction, # but CCD assembly demands a variance plane ampExposure = ccdExposure.Factory(ccdExposure, amp.getRawDataBBox(), afwImage.PARENT) self.updateVariance(ampExposure, amp) ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) ccd = ccdExposure.getDetector() doRotateCalib = False # Rotate calib images for bias/dark/flat correction? nQuarter = ccd.getOrientation().getNQuarter() if nQuarter != 0: doRotateCalib = True if self.config.doDefect: defects = sensorRef.get("defects", immediate=True) self.maskAndInterpDefect(ccdExposure, defects) if self.config.qa.doWriteOss: sensorRef.put(ccdExposure, "ossImage") if self.config.qa.doThumbnailOss: self.writeThumbnail(sensorRef, "ossThumb", ccdExposure) if self.config.doBias: biasExposure = self.getIsrExposure(sensorRef, "bias") if not doRotateCalib: self.biasCorrection(ccdExposure, biasExposure) else: with self.rotated(ccdExposure) as exp: self.biasCorrection(exp, biasExposure) if self.config.doLinearize: self.linearize(ccdExposure) if self.config.doCrosstalk: self.crosstalk.run(ccdExposure) if self.config.doDark: darkExposure = self.getIsrExposure(sensorRef, "dark") if not doRotateCalib: self.darkCorrection(ccdExposure, darkExposure) else: with self.rotated(ccdExposure) as exp: self.darkCorrection(exp, darkExposure) if self.config.doFlat: flatExposure = self.getIsrExposure(sensorRef, "flat") if not doRotateCalib: self.flatCorrection(ccdExposure, flatExposure) else: with self.rotated(ccdExposure) as exp: self.flatCorrection(exp, flatExposure) if self.config.doApplyGains: self.applyGains(ccdExposure, self.config.normalizeGains) if self.config.doWidenSaturationTrails: self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask()) if self.config.doSaturation: self.saturationInterpolation(ccdExposure) if self.config.doFringe: self.fringe.runDataRef(ccdExposure, sensorRef) if self.config.doSetBadRegions: self.setBadRegions(ccdExposure) self.maskAndInterpNan(ccdExposure) if self.config.qa.doWriteFlattened: sensorRef.put(ccdExposure, "flattenedImage") if self.config.qa.doThumbnailFlattened: self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure) self.measureBackground(ccdExposure) if self.config.doGuider: self.guider(ccdExposure) self.roughZeroPoint(ccdExposure) if self.config.doWriteVignettePolygon: self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon) if self.config.doWrite: sensorRef.put(ccdExposure, "postISRCCD") if self._display: im = ccdExposure.getMaskedImage().getImage() im_median = float(numpy.median(im.getArray())) ds9.mtv(im) ds9.scale(min=im_median * 0.95, max=im_median * 1.15) return Struct(exposure=ccdExposure)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """Determine a PSFEX PSF model for an exposure given a list of PSF candidates @param[in] exposure: exposure containing the psf candidates (lsst.afw.image.Exposure) @param[in] psfCandidateList: a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector @param[in,out] metadata a home for interesting tidbits of information @param[in] flagKey: schema key used to mark sources actually used in PSF determination @return psf: a meas.extensions.psfex.PsfexPsf """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = display and \ lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfComponents = display and \ lsstDebug.Info(__name__).displayPsfComponents # show the basis functions showBadCandidates = display and \ lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks) displayResiduals = display and \ lsstDebug.Info(__name__).displayResiduals # show residuals displayPsfMosaic = display and \ lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals # Normalise residuals by object amplitude mi = exposure.getMaskedImage() nCand = len(psfCandidateList) if nCand == 0: raise RuntimeError("No PSF candidates supplied.") # # How big should our PSF models be? # if display: # only needed for debug plots # construct and populate a spatial cell set bbox = mi.getBBox(afwImage.PARENT) psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) else: psfCellSet = None sizes = np.empty(nCand) for i, psfCandidate in enumerate(psfCandidateList): try: if psfCellSet: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) rmsSize = quad.getTraceRadius() sizes[i] = rmsSize if self.config.kernelSize >= 15: self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value") actualKernelSize = int(self.config.kernelSize) else: actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: rms = np.median(sizes) print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms)) # If we manually set the resolution then we need the size in pixel units pixKernelSize = actualKernelSize if self.config.samplingSize > 0: pixKernelSize = int(actualKernelSize*self.config.samplingSize) if pixKernelSize % 2 == 0: pixKernelSize += 1 self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize) psfCandidateList[0].setHeight(pixKernelSize) psfCandidateList[0].setWidth(pixKernelSize) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX # # Insert the good candidates into the set # defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex") args_md = dafBase.PropertySet() args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder)) args_md.set("PSF_SIZE", str(actualKernelSize)) args_md.set("PSF_SAMPLING", str(self.config.samplingSize)) prefs = psfex.Prefs(defaultsFile, args_md) prefs.setCommandLine([]) prefs.addCatalog("psfexPsfDeterminer") prefs.use() principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN) if False else psfex.Context.KEEPHIDDEN) context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(), prefs.getGroupDeg(), principalComponentExclusionFlag) set = psfex.Set(context) set.setVigSize(pixKernelSize, pixKernelSize) set.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes)) set.setRecentroid(self.config.recentroid) catindex, ext = 0, 0 backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue() ccd = exposure.getDetector() if ccd: gain = np.mean(np.array([a.getGain() for a in ccd])) else: gain = 1.0 self.log.warn("Setting gain to %g" % (gain,)) contextvalp = [] for i, key in enumerate(context.getName()): if context.getPcflag(i): contextvalp.append(pcval[pc]) pc += 1 elif key[0] == ':': try: contextvalp.append(exposure.getMetadata().get(key[1:])) except KeyError: raise RuntimeError("*Error*: %s parameter not found in the header of %s" % (key[1:], prefs.getContextName())) else: try: contextvalp.append(np.array([psfCandidateList[_].getSource().get(key) for _ in range(nCand)])) except KeyError: raise RuntimeError("*Error*: %s parameter not found" % (key,)) set.setContextname(i, key) if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits) fluxName = prefs.getPhotfluxRkey() fluxFlagName = "base_" + fluxName + "_flag" xpos, ypos = [], [] for i, psfCandidate in enumerate(psfCandidateList): source = psfCandidate.getSource() xc, yc = source.getX(), source.getY() try: int(xc), int(yc) except ValueError: continue try: pstamp = psfCandidate.getMaskedImage().clone() except Exception as e: continue if fluxFlagName in source.schema and source.get(fluxFlagName): continue flux = source.get(fluxName) if flux < 0 or np.isnan(flux): continue # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate). # Having created the sample, we must proceed to configure it, and then fini (finalize), # or it will be malformed. try: sample = set.newSample() sample.setCatindex(catindex) sample.setExtindex(ext) sample.setObjindex(i) imArray = pstamp.getImage().getArray() imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \ -2*psfex.cvar.BIG sample.setVig(imArray) sample.setNorm(flux) sample.setBacknoise2(backnoise2) sample.setGain(gain) sample.setX(xc) sample.setY(yc) sample.setFluxrad(sizes[i]) for j in range(set.getNcontext()): sample.setContext(j, float(contextvalp[j][i])) except Exception as e: self.log.debug("Exception when processing sample at (%f,%f): %s", xc, yc, e) continue else: set.finiSample(sample) xpos.append(xc) # for QA ypos.append(yc) if displayExposure: with ds9.Buffering(): ds9.dot("o", xc, yc, ctype=ds9.CYAN, size=4, frame=frame) if set.getNsample() == 0: raise RuntimeError("No good PSF candidates to pass to PSFEx") #---- Update min and max and then the scaling for i in range(set.getNcontext()): cmin = contextvalp[i].min() cmax = contextvalp[i].max() set.setContextScale(i, cmax - cmin) set.setContextOffset(i, (cmin + cmax)/2.0) # Don't waste memory! set.trimMemory() #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX # # Do a PSFEX decomposition of those PSF candidates # fields = psfex.vectorField() field = psfex.Field("Unknown") field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample()) field.finalize() fields.append(field) sets = psfex.vectorSet() sets.append(set) psfex.makeit(fields, sets) psfs = field.getPsfs() # Flag which objects were actually used in psfex by good_indices = [] for i in range(sets[0].getNsample()): index = sets[0].getSample(i).getObjindex() if index > -1: good_indices.append(index) if flagKey is not None: for i, psfCandidate in enumerate(psfCandidateList): source = psfCandidate.getSource() if i in good_indices: source.set(flagKey, True) xpos = np.array(xpos) ypos = np.array(ypos) numGoodStars = len(good_indices) avgX, avgY = np.mean(xpos), np.mean(ypos) psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY)) if False and (displayResiduals or displayPsfMosaic): ext = 0 frame = 1 diagnostics = True catDir = "." title = "psfexPsfDeterminer" psfex.psfex.showPsf(psfs, set, ext, [(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())], nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir, title=title) # # Display code for debugging # if display: assert psfCellSet is not None if displayExposure: maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame) if displayResiduals: maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: maUtils.showPsf(psf, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) # # Generate some QA information # # Count PSF stars # if metadata is not None: metadata.set("spatialFitChi2", np.nan) metadata.set("numAvailStars", nCand) metadata.set("numGoodStars", numGoodStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psfCellSet = None return psf, psfCellSet