def run(display=False): exposure = loadData() schema = afwTable.SourceTable.makeMinimalSchema() # # Create the detection task # config = SourceDetectionTask.ConfigClass() config.thresholdPolarity = "both" config.background.isNanSafe = True config.thresholdValue = 3 detectionTask = SourceDetectionTask(config=config, schema=schema) # # And the measurement Task # config = SingleFrameMeasurementTask.ConfigClass() config.algorithms.names = ["base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux"] config.algorithms["base_CircularApertureFlux"].radii = [1, 2, 4, 8, 12, 16] # pixels config.slots.gaussianFlux = None config.slots.modelFlux = None config.slots.psfFlux = None algMetadata = dafBase.PropertyList() measureTask = SingleFrameMeasurementTask(schema, algMetadata=algMetadata, config=config) radii = algMetadata.getArray("base_CircularApertureFlux_radii") # # Create the output table # tab = afwTable.SourceTable.make(schema) # # Process the data # result = detectionTask.run(tab, exposure) sources = result.sources print("Found %d sources (%d +ve, %d -ve)" % (len(sources), result.fpSets.numPos, result.fpSets.numNeg)) measureTask.run(sources, exposure) if display: # display image (see also --debug argparse option) afwDisplay.setDefaultMaskTransparency(75) frame = 1 disp = afwDisplay.Display(frame=frame) disp.mtv(exposure) with disp.Buffering(): for s in sources: xy = s.getCentroid() disp.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) disp.dot(s.getShape(), *xy, ctype=afwDisplay.RED) for radius in radii: disp.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
def display(self, exposure, results, convolvedImage=None): """Display detections if so configured Displays the ``exposure`` in frame 0, overlays the detection peaks. Requires that ``lsstDebug`` has been set up correctly, so that ``lsstDebug.Info("lsst.meas.algorithms.detection")`` evaluates `True`. If the ``convolvedImage`` is non-`None` and ``lsstDebug.Info("lsst.meas.algorithms.detection") > 1``, the ``convolvedImage`` will be displayed in frame 1. Parameters ---------- exposure : `lsst.afw.image.Exposure` Exposure to display, on which will be plotted the detections. results : `lsst.pipe.base.Struct` Results of the 'detectFootprints' method, containing positive and negative footprints (which contain the peak positions that we will plot). This is a `Struct` with ``positive`` and ``negative`` elements that are of type `lsst.afw.detection.FootprintSet`. convolvedImage : `lsst.afw.image.Image`, optional Convolved image used for thresholding. """ try: import lsstDebug display = lsstDebug.Info(__name__).display except ImportError: try: display except NameError: display = False if not display: return afwDisplay.setDefaultMaskTransparency(75) disp0 = afwDisplay.Display(frame=0) disp0.mtv(exposure, title="detection") def plotPeaks(fps, ctype): if fps is None: return with disp0.Buffering(): for fp in fps.getFootprints(): for pp in fp.getPeaks(): disp0.dot("+", pp.getFx(), pp.getFy(), ctype=ctype) plotPeaks(results.positive, "yellow") plotPeaks(results.negative, "red") if convolvedImage and display > 1: disp1 = afwDisplay.Display(frame=1) disp1.mtv(convolvedImage, title="PSF smoothed")
def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground): """Provide visualization of the inputs and ouputs to the Psf-matching code Parameters ---------- kernelCellSet : `lsst.afw.math.SpatialCellSet` The SpatialCellSet used in determining the matching kernel and background spatialKernel : `lsst.afw.math.LinearCombinationKernel` Spatially varying Psf-matching kernel spatialBackground : `lsst.afw.math.Function2D` Spatially varying background-matching function """ import lsstDebug displayCandidates = lsstDebug.Info(__name__).displayCandidates displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel showBadCandidates = lsstDebug.Info(__name__).showBadCandidates maskTransparency = lsstDebug.Info(__name__).maskTransparency if not maskTransparency: maskTransparency = 0 afwDisplay.setDefaultMaskTransparency(maskTransparency) if displayCandidates: diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, frame=lsstDebug.frame, showBadCandidates=showBadCandidates) lsstDebug.frame += 1 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, frame=lsstDebug.frame, showBadCandidates=showBadCandidates, kernels=True) lsstDebug.frame += 1 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground, frame=lsstDebug.frame, showBadCandidates=showBadCandidates, resids=True) lsstDebug.frame += 1 if displayKernelBasis: diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame) lsstDebug.frame += 1 if displayKernelMosaic: diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame) lsstDebug.frame += 1 if plotKernelSpatialModel: diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
def displayImages(root): """Display coadd images in different frames, with the bounding boxes of the observations that went into them overlayed. """ import lsst.afw.display as afwDisplay afwDisplay.setDefaultMaskTransparency(75) butler = lsst.daf.persistence.Butler(root=root) skyMap = butler.get("deepCoadd_skyMap") tractInfo = skyMap[0] task = lsst.pipe.tasks.mocks.MockCoaddTask() coadds = [patchRef.get("deepCoadd", immediate=True) for patchRef in task.iterPatchRefs(butler, tractInfo)] for n, coadd in enumerate(coadds): afwDisplay.Display(frame=n + 1).mtv(coadd, title="displayImages: coadd") for n, coadd in enumerate(coadds): afwDisplay.utils.drawCoaddInputs(coadd, frame=n + 1) return butler
def subtractExposures(self, templateExposure, scienceExposure, templateFwhmPix=None, scienceFwhmPix=None, candidateList=None, doWarping=True, convolveTemplate=True): """Register, Psf-match and subtract two Exposures. Do the following, in order: - Warp templateExposure to match scienceExposure, if their WCSs do not already match - Determine a PSF matching kernel and differential background model that matches templateExposure to scienceExposure - PSF-match templateExposure to scienceExposure - Compute subtracted exposure (see return values for equation). Parameters ---------- templateExposure : `lsst.afw.image.Exposure` Exposure to PSF-match to scienceExposure scienceExposure : `lsst.afw.image.Exposure` Reference Exposure templateFwhmPix : `float` FWHM (in pixels) of the Psf in the template image (image to convolve) scienceFwhmPix : `float` FWHM (in pixels) of the Psf in the science image candidateList : `list`, optional A list of footprints/maskedImages for kernel candidates; if `None` then source detection is run. - Currently supported: list of Footprints or measAlg.PsfCandidateF doWarping : `bool` What to do if ``templateExposure``` and ``scienceExposure`` WCSs do not match: - if `True` then warp ``templateExposure`` to match ``scienceExposure`` - if `False` then raise an Exception convolveTemplate : `bool` Convolve the template image or the science image - if `True`, ``templateExposure`` is warped if doWarping, ``templateExposure`` is convolved - if `False`, ``templateExposure`` is warped if doWarping, ``scienceExposure is`` convolved Returns ------- result : `lsst.pipe.base.Struct` An `lsst.pipe.base.Struct` containing these fields: - ``subtractedExposure`` : subtracted Exposure scienceExposure - (matchedImage + backgroundModel) - ``matchedImage`` : ``templateExposure`` after warping to match ``templateExposure`` (if doWarping true), and convolving with psfMatchingKernel - ``psfMatchingKernel`` : PSF matching kernel - ``backgroundModel`` : differential background model - ``kernelCellSet`` : SpatialCellSet used to determine PSF matching kernel """ results = self.matchExposures( templateExposure=templateExposure, scienceExposure=scienceExposure, templateFwhmPix=templateFwhmPix, scienceFwhmPix=scienceFwhmPix, candidateList=candidateList, doWarping=doWarping, convolveTemplate=convolveTemplate ) subtractedExposure = afwImage.ExposureF(scienceExposure, True) if convolveTemplate: subtractedMaskedImage = subtractedExposure.getMaskedImage() subtractedMaskedImage -= results.matchedExposure.getMaskedImage() subtractedMaskedImage -= results.backgroundModel else: subtractedExposure.setMaskedImage(results.warpedExposure.getMaskedImage()) subtractedMaskedImage = subtractedExposure.getMaskedImage() subtractedMaskedImage -= results.matchedExposure.getMaskedImage() subtractedMaskedImage -= results.backgroundModel # Preserve polarity of differences subtractedMaskedImage *= -1 # Place back on native photometric scale subtractedMaskedImage /= results.psfMatchingKernel.computeImage( afwImage.ImageD(results.psfMatchingKernel.getDimensions()), False) import lsstDebug display = lsstDebug.Info(__name__).display displayDiffIm = lsstDebug.Info(__name__).displayDiffIm maskTransparency = lsstDebug.Info(__name__).maskTransparency if not maskTransparency: maskTransparency = 0 if display: afwDisplay.setDefaultMaskTransparency(maskTransparency) if display and displayDiffIm: disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(templateExposure, title="Template") lsstDebug.frame += 1 disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(results.matchedExposure, title="Matched template") lsstDebug.frame += 1 disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(scienceExposure, title="Science Image") lsstDebug.frame += 1 disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(subtractedExposure, title="Difference Image") lsstDebug.frame += 1 results.subtractedExposure = subtractedExposure return results
def matchMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList, templateFwhmPix=None, scienceFwhmPix=None): """PSF-match a MaskedImage (templateMaskedImage) to a reference MaskedImage (scienceMaskedImage). Do the following, in order: - Determine a PSF matching kernel and differential background model that matches templateMaskedImage to scienceMaskedImage - Convolve templateMaskedImage by the PSF matching kernel Parameters ---------- templateMaskedImage : `lsst.afw.image.MaskedImage` masked image to PSF-match to the reference masked image; must be warped to match the reference masked image scienceMaskedImage : `lsst.afw.image.MaskedImage` maskedImage whose PSF is to be matched to templateFwhmPix : `float` FWHM (in pixels) of the Psf in the template image (image to convolve) scienceFwhmPix : `float` FWHM (in pixels) of the Psf in the science image candidateList : `list`, optional A list of footprints/maskedImages for kernel candidates; if `None` then source detection is run. - Currently supported: list of Footprints or measAlg.PsfCandidateF Returns ------- result : `callable` An `lsst.pipe.base.Struct` containing these fields: - psfMatchedMaskedImage: the PSF-matched masked image = ``templateMaskedImage`` convolved with psfMatchingKernel. This has the same xy0, dimensions and wcs as ``scienceMaskedImage``. - psfMatchingKernel: the PSF matching kernel - backgroundModel: differential background model - kernelCellSet: SpatialCellSet used to solve for the PSF matching kernel Raises ------ RuntimeError Raised if input images have different dimensions """ import lsstDebug display = lsstDebug.Info(__name__).display displayTemplate = lsstDebug.Info(__name__).displayTemplate displaySciIm = lsstDebug.Info(__name__).displaySciIm displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells maskTransparency = lsstDebug.Info(__name__).maskTransparency if not maskTransparency: maskTransparency = 0 if display: afwDisplay.setDefaultMaskTransparency(maskTransparency) if not candidateList: raise RuntimeError("Candidate list must be populated by makeCandidateList") if not self._validateSize(templateMaskedImage, scienceMaskedImage): self.log.error("ERROR: Input images different size") raise RuntimeError("Input images different size") if display and displayTemplate: disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(templateMaskedImage, title="Image to convolve") lsstDebug.frame += 1 if display and displaySciIm: disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(scienceMaskedImage, title="Image to not convolve") lsstDebug.frame += 1 kernelCellSet = self._buildCellSet(templateMaskedImage, scienceMaskedImage, candidateList) if display and displaySpatialCells: diffimUtils.showKernelSpatialCells(scienceMaskedImage, kernelCellSet, symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=4, frame=lsstDebug.frame, title="Image to not convolve") lsstDebug.frame += 1 if templateFwhmPix and scienceFwhmPix: self.log.info("Matching Psf FWHM %.2f -> %.2f pix", templateFwhmPix, scienceFwhmPix) if self.kConfig.useBicForKernelBasis: tmpKernelCellSet = self._buildCellSet(templateMaskedImage, scienceMaskedImage, candidateList) nbe = diffimTools.NbasisEvaluator(self.kConfig, templateFwhmPix, scienceFwhmPix) bicDegrees = nbe(tmpKernelCellSet, self.log) basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix, alardDegGauss=bicDegrees[0], metadata=self.metadata) del tmpKernelCellSet else: basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix, metadata=self.metadata) spatialSolution, psfMatchingKernel, backgroundModel = self._solve(kernelCellSet, basisList) psfMatchedMaskedImage = afwImage.MaskedImageF(templateMaskedImage.getBBox()) doNormalize = False afwMath.convolve(psfMatchedMaskedImage, templateMaskedImage, psfMatchingKernel, doNormalize) return pipeBase.Struct( matchedImage=psfMatchedMaskedImage, psfMatchingKernel=psfMatchingKernel, backgroundModel=backgroundModel, kernelCellSet=kernelCellSet, )
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """Determine a PCA PSF model for an exposure given a list of PSF candidates. Parameters ---------- exposure : `lsst.afw.image.Exposure` Exposure containing the psf candidates. psfCandidateList : `list` of `lsst.meas.algorithms.PsfCandidate` A sequence of PSF candidates typically obtained by detecting sources and then running them through a star selector. metadata : `lsst.daf.base import PropertyList` or `None`, optional A home for interesting tidbits of information. flagKey : `str`, optional Schema key used to mark sources actually used in PSF determination. Returns ------- psf : `lsst.meas.algorithms.PcaPsf` The measured PSF. psfCellSet : `lsst.afw.math.SpatialCellSet` The PSF candidates. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info(__name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) # match Kernel amplitudes for spatial plots matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes # Keep matplotlib alive post mortem keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # Include bad candidates # Normalize residuals by object amplitude normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals pause = lsstDebug.Info(__name__).pause # Prompt user after each iteration? if display: afwDisplay.setDefaultMaskTransparency(75) if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwGeom.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.log.warn("WARNING: NOT scaling kernelSize by stellar quadrupole moment " "because config.kernelSize=%s >= 15; " "using config.kernelSize as as the width, instead", self.config.kernelSize) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2*int(self.config.kernelSize*math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print("Median size=%s" % (medSize,)) self.log.trace("Kernel size=%s", actualKernelSize) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print("Removing %d blended Psf candidates" % len(blendedCandidates)) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: if displayExposure: disp = afwDisplay.Display(frame=0) disp.mtv(exposure, title="psf determination") utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW, size=4, display=disp) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iterNum in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates): # maybe include bad candidates try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append((im, "%d%s" % (utils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception: continue if len(stamps) == 0: print("WARNING: No PSF candidates to show; try setting showBadCandidates=True") else: mos = afwDisplay.utils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append(im, label, (afwDisplay.GREEN if status == afwMath.SpatialCellCandidate.GOOD else afwDisplay.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else afwDisplay.RED)) disp8 = afwDisplay.Display(frame=8) mos.makeMosaic(display=disp8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.log.debug("chi^2=%s; id=%s", cand.getChi2(), cand.getSource().getId()) for cand in awfulCandidates: if display: print("Removing bad candidate: id=%d, chi^2=%f" % (cand.getSource().getId(), cand.getChi2())) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates rchi2 = cand.getChi2() # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = psf.getKernel() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception: continue fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p*k.getSum() predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters())] residuals.append([a/amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:, k].mean() rms = residuals[:, k].std() elif False: # Using interquartile range sr = numpy.sort(residuals[:, k]) mean = (sr[int(0.5*len(sr))] if len(sr)%2 else 0.5*(sr[int(0.5*len(sr))] + sr[int(0.5*len(sr)) + 1])) rms = 0.74*(sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))]) else: stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP) mean = stats.getValue(afwMath.MEANCLIP) rms = stats.getValue(afwMath.STDEVCLIP) rms = max(1.0e-4, rms) # Don't trust RMS below this due to numerical issues if display: print("Mean for component %d is %f" % (k, mean)) print("RMS for component %d is %f" % (k, rms)) badCandidates = list() for i, cand in enumerate(candidates): if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject*rms: badCandidates.append(i) badCandidates.sort(key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates): cand = candidates[c] if display: print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k, residuals[badCandidates[i], k], self.config.spatialReject*rms)) cand.setStatus(afwMath.SpatialCellCandidate.BAD) # # Display results # if display and displayIterations: if displayExposure: if iterNum > 0: disp.erase() utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", size=8, display=disp, ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, ctypeUnused=afwDisplay.MAGENTA) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", size=10, display=disp, ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED) if displayResiduals: while True: try: disp4 = afwDisplay.Display(frame=4) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) disp5 = afwDisplay.Display(frame=5) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except Exception: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) utils.showPsf(psf, eigenValues, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp7.scale('linear', 0, 1) if displayPsfSpatialModel: utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = input("Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:] else: reply = "" if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"): if reply == "c": pause = False elif reply == "h": print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " "s[ave fileName] y[es]") continue elif reply == "p": import pdb pdb.set_trace() elif reply == "q": display = False elif reply == "Q": sys.exit(1) elif reply == "s": fileName = args.pop(0) if not fileName: print("Please provide a filename") continue print("Saving to %s" % fileName) utils.saveSpatialCellSet(psfCellSet, fileName=fileName) continue break else: print("Unrecognised response: %s" % reply, file=sys.stderr) if reply == "n": break # One last time, to take advantage of the last iteration psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # Display code for debugging # if display and reply != "n": disp = afwDisplay.Display(frame=0) if displayExposure: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=8, display=disp) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=10, display=disp) if displayResiduals: disp4 = afwDisplay.Display(frame=4) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) utils.showPsf(psf, eigenValues, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp7.scale("linear", 0, 1) if displayPsfSpatialModel: utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) # # Generate some QA information # # Count PSF stars # numGoodStars = 0 numAvailStars = 0 avgX = 0.0 avgY = 0.0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # don't ignore BAD stars numAvailStars += 1 for cand in cell.begin(True): # do ignore BAD stars src = cand.getSource() if flagKey is not None: src.set(flagKey, True) avgX += src.getX() avgY += src.getY() numGoodStars += 1 avgX /= numGoodStars avgY /= numGoodStars if metadata is not None: metadata.set("spatialFitChi2", fitChi2) metadata.set("numGoodStars", numGoodStars) metadata.set("numAvailStars", numAvailStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psf = PcaPsf(psf.getKernel(), lsst.geom.Point2D(avgX, avgY)) return psf, psfCellSet
import lsst.utils import lsst.utils.tests import lsst.geom import lsst.afw.image as afwImage import lsst.afw.math as afwMath import lsst.afw.math.detail as mathDetail import lsst.pex.exceptions as pexExcept from test_kernel import makeDeltaFunctionKernelList, makeGaussianKernelList from lsst.log import Log import lsst.afw.display as afwDisplay Log.getLogger("afw.image.Mask").setLevel(Log.INFO) afwDisplay.setDefaultMaskTransparency(75) try: display except NameError: display = False try: dataDir = os.path.join(lsst.utils.getPackageDir("afwdata"), "data") except pexExcept.NotFoundError: dataDir = None else: InputMaskedImagePath = os.path.join(dataDir, "medexp.fits") FullMaskedImage = afwImage.MaskedImageF(InputMaskedImagePath) # input image contains a saturated star, a bad column, and a faint star
def matchMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList, templateFwhmPix=None, scienceFwhmPix=None): """PSF-match a MaskedImage (templateMaskedImage) to a reference MaskedImage (scienceMaskedImage). Do the following, in order: - Determine a PSF matching kernel and differential background model that matches templateMaskedImage to scienceMaskedImage - Convolve templateMaskedImage by the PSF matching kernel Parameters ---------- templateMaskedImage : `lsst.afw.image.MaskedImage` masked image to PSF-match to the reference masked image; must be warped to match the reference masked image scienceMaskedImage : `lsst.afw.image.MaskedImage` maskedImage whose PSF is to be matched to templateFwhmPix : `float` FWHM (in pixels) of the Psf in the template image (image to convolve) scienceFwhmPix : `float` FWHM (in pixels) of the Psf in the science image candidateList : `list`, optional A list of footprints/maskedImages for kernel candidates; if `None` then source detection is run. - Currently supported: list of Footprints or measAlg.PsfCandidateF Returns ------- result : `callable` An `lsst.pipe.base.Struct` containing these fields: - psfMatchedMaskedImage: the PSF-matched masked image = ``templateMaskedImage`` convolved with psfMatchingKernel. This has the same xy0, dimensions and wcs as ``scienceMaskedImage``. - psfMatchingKernel: the PSF matching kernel - backgroundModel: differential background model - kernelCellSet: SpatialCellSet used to solve for the PSF matching kernel Raises ------ RuntimeError Raised if input images have different dimensions """ import lsstDebug display = lsstDebug.Info(__name__).display displayTemplate = lsstDebug.Info(__name__).displayTemplate displaySciIm = lsstDebug.Info(__name__).displaySciIm displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells maskTransparency = lsstDebug.Info(__name__).maskTransparency if not maskTransparency: maskTransparency = 0 if display: afwDisplay.setDefaultMaskTransparency(maskTransparency) if not candidateList: raise RuntimeError("Candidate list must be populated by makeCandidateList") if not self._validateSize(templateMaskedImage, scienceMaskedImage): self.log.error("ERROR: Input images different size") raise RuntimeError("Input images different size") if display and displayTemplate: disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(templateMaskedImage, title="Image to convolve") lsstDebug.frame += 1 if display and displaySciIm: disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(scienceMaskedImage, title="Image to not convolve") lsstDebug.frame += 1 kernelCellSet = self._buildCellSet(templateMaskedImage, scienceMaskedImage, candidateList) if display and displaySpatialCells: diffimUtils.showKernelSpatialCells(scienceMaskedImage, kernelCellSet, symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=4, frame=lsstDebug.frame, title="Image to not convolve") lsstDebug.frame += 1 if templateFwhmPix and scienceFwhmPix: self.log.info("Matching Psf FWHM %.2f -> %.2f pix", templateFwhmPix, scienceFwhmPix) if self.kConfig.useBicForKernelBasis: tmpKernelCellSet = self._buildCellSet(templateMaskedImage, scienceMaskedImage, candidateList) nbe = diffimTools.NbasisEvaluator(self.kConfig, templateFwhmPix, scienceFwhmPix) bicDegrees = nbe(tmpKernelCellSet, self.log) basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix, alardDegGauss=bicDegrees[0], metadata=self.metadata) del tmpKernelCellSet else: basisList = makeKernelBasisList(self.kConfig, templateFwhmPix, scienceFwhmPix, metadata=self.metadata) spatialSolution, psfMatchingKernel, backgroundModel = self._solve(kernelCellSet, basisList) psfMatchedMaskedImage = afwImage.MaskedImageF(templateMaskedImage.getBBox()) convolutionControl = afwMath.ConvolutionControl() convolutionControl.setDoNormalize(False) afwMath.convolve(psfMatchedMaskedImage, templateMaskedImage, psfMatchingKernel, convolutionControl) return pipeBase.Struct( matchedImage=psfMatchedMaskedImage, psfMatchingKernel=psfMatchingKernel, backgroundModel=backgroundModel, kernelCellSet=kernelCellSet, )
def selectSources(self, sourceCat, matches=None, exposure=None): """Return a selection of psf-like objects. Parameters ---------- sourceCat : `lsst.afw.table.SourceCatalog` Catalog of sources to select from. This catalog must be contiguous in memory. matches : `list` of `lsst.afw.table.ReferenceMatch` or None Ignored by this source selector. exposure : `lsst.afw.image.Exposure` or None The exposure the catalog was built from; used for debug display. Return ------ struct : `lsst.pipe.base.Struct` The struct contains the following data: - selected : `numpy.ndarray` of `bool`` Boolean array of sources that were selected, same length as sourceCat. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = display and \ lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells plotFwhmHistogram = display and plt and \ lsstDebug.Info(__name__).plotFwhmHistogram # Plot histogram of FWHM plotFlags = display and plt and \ lsstDebug.Info(__name__).plotFlags # Plot the sources coloured by their flags plotRejection = display and plt and \ lsstDebug.Info(__name__).plotRejection # Plot why sources are rejected afwDisplay.setDefaultMaskTransparency(75) fluxName = self.config.fluxName fluxErrName = self.config.fluxErrName minFwhm = self.config.minFwhm maxFwhm = self.config.maxFwhm maxFwhmVariability = self.config.maxFwhmVariability maxbad = self.config.maxbad maxbadflag = self.config.maxbadflag maxellip = self.config.maxellip minsn = self.config.minsn maxelong = (maxellip + 1.0) / (1.0 - maxellip) if maxellip < 1.0 else 100 # Unpack the catalogue shape = sourceCat.getShapeDefinition() ixx = sourceCat.get("%s.xx" % shape) iyy = sourceCat.get("%s.yy" % shape) fwhm = 2 * np.sqrt(2 * np.log(2)) * np.sqrt(0.5 * (ixx + iyy)) elong = 0.5 * (ixx - iyy) / (ixx + iyy) flux = sourceCat.get(fluxName) fluxErr = sourceCat.get(fluxErrName) sn = flux / np.where(fluxErr > 0, fluxErr, 1) sn[fluxErr <= 0] = -psfexLib.BIG flags = 0x0 for i, f in enumerate(self.config.badFlags): flags = np.bitwise_or(flags, np.where(sourceCat.get(f), 1 << i, 0)) # # Estimate the acceptable range of source widths # good = np.logical_and(sn > minsn, np.logical_not(flags)) good = np.logical_and(good, elong < maxelong) good = np.logical_and(good, fwhm >= minFwhm) good = np.logical_and(good, fwhm < maxFwhm) fwhmMode, fwhmMin, fwhmMax = compute_fwhmrange( fwhm[good], maxFwhmVariability, minFwhm, maxFwhm, plot=dict(fwhmHistogram=plotFwhmHistogram)) # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Here's select_candidates # # ---- Apply some selection over flags, fluxes... bad = (flags != 0) # set.setBadFlags(int(sum(bad))) if plotRejection: selectionVectors = [] selectionVectors.append((bad, "flags %d" % sum(bad))) dbad = sn < minsn # set.setBadSN(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "S/N %d" % sum(dbad))) dbad = fwhm < fwhmMin # set.setBadFrmin(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "fwhmMin %d" % sum(dbad))) dbad = fwhm > fwhmMax # set.setBadFrmax(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "fwhmMax %d" % sum(dbad))) dbad = elong > maxelong # set.setBadElong(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "elong %d" % sum(dbad))) # -- ... and check the integrity of the sample if maxbadflag: nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet]) dbad = nbad > maxbad # set.setBadPix(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "badpix %d" % sum(dbad))) good = np.logical_not(bad) # # We know enough to plot, if so requested # frame = 0 if displayExposure: mi = exposure.getMaskedImage() disp = afwDisplay.Display(frame=frame) disp.mtv(mi, title="PSF candidates") with disp.Buffering(): for i, source in enumerate(sourceCat): if good[i]: ctype = afwDisplay.GREEN # star candidate else: ctype = afwDisplay.RED # not star disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) if plotFlags or plotRejection: imag = -2.5 * np.log10(flux) plt.clf() alpha = 0.5 if plotFlags: isSet = np.where(flags == 0x0)[0] plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label="good") for i, f in enumerate(self.config.badFlags): mask = 1 << i isSet = np.where(np.bitwise_and(flags, mask))[0] if isSet.any(): if np.isfinite(imag[isSet] + fwhm[isSet]).any(): label = re.sub( r"\_flag", "", re.sub( r"^base\_", "", re.sub(r"^.*base\_PixelFlags\_flag\_", "", f))) plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label=label) else: for bad, label in selectionVectors: plt.plot(imag[bad], fwhm[bad], 'o', alpha=alpha, label=label) plt.plot(imag[good], fwhm[good], 'o', color="black", label="selected") [plt.axhline(_, color='red') for _ in [fwhmMin, fwhmMax]] plt.xlim(np.median(imag[good]) + 5 * np.array([-1, 1])) plt.ylim(fwhm[np.where(np.isfinite(fwhm + imag))].min(), 2 * fwhmMax) plt.legend(loc=2) plt.xlabel("Instrumental %s Magnitude" % fluxName.split(".")[-1].title()) plt.ylabel("fwhm") title = "PSFEX Star Selection" plt.title("%s %d selected" % (title, sum(good))) if displayExposure: global eventHandler eventHandler = EventHandler(plt.axes(), imag, fwhm, sourceCat.getX(), sourceCat.getY(), frames=[frame]) if plotFlags or plotRejection: while True: try: reply = input( "continue? [y[es] h(elp) p(db) q(uit)] ").strip() except EOFError: reply = "y" if not reply: reply = "y" if reply[0] == "h": print("""\ At this prompt, you can continue with almost any key; 'p' enters pdb, 'q' returns to the shell, and 'h' prints this text """, end=' ') if displayExposure: print(""" If you put the cursor on a point in the matplotlib scatter plot and hit 'p' you'll see it in ds9.""" ) elif reply[0] == "p": import pdb pdb.set_trace() elif reply[0] == 'q': sys.exit(1) else: break return Struct(selected=good)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """Determine a PCA PSF model for an exposure given a list of PSF candidates. Parameters ---------- exposure : `lsst.afw.image.Exposure` Exposure containing the psf candidates. psfCandidateList : `list` of `lsst.meas.algorithms.PsfCandidate` A sequence of PSF candidates typically obtained by detecting sources and then running them through a star selector. metadata : `lsst.daf.base import PropertyList` or `None`, optional A home for interesting tidbits of information. flagKey : `str`, optional Schema key used to mark sources actually used in PSF determination. Returns ------- psf : `lsst.meas.algorithms.PcaPsf` The measured PSF. psfCellSet : `lsst.afw.math.SpatialCellSet` The PSF candidates. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info(__name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) # match Kernel amplitudes for spatial plots matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes # Keep matplotlib alive post mortem keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # Include bad candidates # Normalize residuals by object amplitude normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals pause = lsstDebug.Info(__name__).pause # Prompt user after each iteration? if display: afwDisplay.setDefaultMaskTransparency(75) if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwGeom.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.log.warning("WARNING: NOT scaling kernelSize by stellar quadrupole moment " "because config.kernelSize=%s >= 15; " "using config.kernelSize as the width, instead", self.config.kernelSize) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2*int(self.config.kernelSize*math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print("Median size=%s" % (medSize,)) self.log.trace("Kernel size=%s", actualKernelSize) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print("Removing %d blended Psf candidates" % len(blendedCandidates)) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: if displayExposure: disp = afwDisplay.Display(frame=0) disp.mtv(exposure, title="psf determination") utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW, size=4, display=disp) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iterNum in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates): # maybe include bad candidates try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append((im, "%d%s" % (utils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception: continue if len(stamps) == 0: print("WARNING: No PSF candidates to show; try setting showBadCandidates=True") else: mos = afwDisplay.utils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append(im, label, (afwDisplay.GREEN if status == afwMath.SpatialCellCandidate.GOOD else afwDisplay.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else afwDisplay.RED)) disp8 = afwDisplay.Display(frame=8) mos.makeMosaic(display=disp8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.log.debug("chi^2=%s; id=%s", cand.getChi2(), cand.getSource().getId()) for cand in awfulCandidates: if display: print("Removing bad candidate: id=%d, chi^2=%f" % (cand.getSource().getId(), cand.getChi2())) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates rchi2 = cand.getChi2() # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = psf.getKernel() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception: continue fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p*k.getSum() predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters())] residuals.append([a/amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:, k].mean() rms = residuals[:, k].std() elif False: # Using interquartile range sr = numpy.sort(residuals[:, k]) mean = (sr[int(0.5*len(sr))] if len(sr)%2 else 0.5*(sr[int(0.5*len(sr))] + sr[int(0.5*len(sr)) + 1])) rms = 0.74*(sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))]) else: stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP) mean = stats.getValue(afwMath.MEANCLIP) rms = stats.getValue(afwMath.STDEVCLIP) rms = max(1.0e-4, rms) # Don't trust RMS below this due to numerical issues if display: print("Mean for component %d is %f" % (k, mean)) print("RMS for component %d is %f" % (k, rms)) badCandidates = list() for i, cand in enumerate(candidates): if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject*rms: badCandidates.append(i) badCandidates.sort(key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates): cand = candidates[c] if display: print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k, residuals[badCandidates[i], k], self.config.spatialReject*rms)) cand.setStatus(afwMath.SpatialCellCandidate.BAD) # # Display results # if display and displayIterations: if displayExposure: if iterNum > 0: disp.erase() utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", size=8, display=disp, ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, ctypeUnused=afwDisplay.MAGENTA) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", size=10, display=disp, ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED) if displayResiduals: while True: try: disp4 = afwDisplay.Display(frame=4) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) disp5 = afwDisplay.Display(frame=5) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except Exception: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) utils.showPsf(psf, eigenValues, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp7.scale('linear', 0, 1) if displayPsfSpatialModel: utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = input("Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:] else: reply = "" if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"): if reply == "c": pause = False elif reply == "h": print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " "s[ave fileName] y[es]") continue elif reply == "p": import pdb pdb.set_trace() elif reply == "q": display = False elif reply == "Q": sys.exit(1) elif reply == "s": fileName = args.pop(0) if not fileName: print("Please provide a filename") continue print("Saving to %s" % fileName) utils.saveSpatialCellSet(psfCellSet, fileName=fileName) continue break else: print("Unrecognised response: %s" % reply, file=sys.stderr) if reply == "n": break # One last time, to take advantage of the last iteration psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # Display code for debugging # if display and reply != "n": disp = afwDisplay.Display(frame=0) if displayExposure: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=8, display=disp) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=10, display=disp) if displayResiduals: disp4 = afwDisplay.Display(frame=4) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) utils.showPsf(psf, eigenValues, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp7.scale("linear", 0, 1) if displayPsfSpatialModel: utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) # # Generate some QA information # # Count PSF stars # numGoodStars = 0 numAvailStars = 0 avgX = 0.0 avgY = 0.0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # don't ignore BAD stars numAvailStars += 1 for cand in cell.begin(True): # do ignore BAD stars src = cand.getSource() if flagKey is not None: src.set(flagKey, True) avgX += src.getX() avgY += src.getY() numGoodStars += 1 avgX /= numGoodStars avgY /= numGoodStars if metadata is not None: metadata["spatialFitChi2"] = fitChi2 metadata["numGoodStars"] = numGoodStars metadata["numAvailStars"] = numAvailStars metadata["avgX"] = avgX metadata["avgY"] = avgY psf = PcaPsf(psf.getKernel(), lsst.geom.Point2D(avgX, avgY)) return psf, psfCellSet
def subtractMaskedImages(self, templateMaskedImage, scienceMaskedImage, candidateList, templateFwhmPix=None, scienceFwhmPix=None): """Psf-match and subtract two MaskedImages. Do the following, in order: - PSF-match templateMaskedImage to scienceMaskedImage - Determine the differential background - Return the difference: scienceMaskedImage ((warped templateMaskedImage convolved with psfMatchingKernel) + backgroundModel) Parameters ---------- templateMaskedImage : `lsst.afw.image.MaskedImage` MaskedImage to PSF-match to ``scienceMaskedImage`` scienceMaskedImage : `lsst.afw.image.MaskedImage` Reference MaskedImage templateFwhmPix : `float` FWHM (in pixels) of the Psf in the template image (image to convolve) scienceFwhmPix : `float` FWHM (in pixels) of the Psf in the science image candidateList : `list`, optional A list of footprints/maskedImages for kernel candidates; if `None` then source detection is run. - Currently supported: list of Footprints or measAlg.PsfCandidateF Returns ------- results : `lsst.pipe.base.Struct` An `lsst.pipe.base.Struct` containing these fields: - ``subtractedMaskedImage`` : ``scienceMaskedImage`` - (matchedImage + backgroundModel) - ``matchedImage`` : templateMaskedImage convolved with psfMatchingKernel - `psfMatchingKernel`` : PSF matching kernel - ``backgroundModel`` : differential background model - ``kernelCellSet`` : SpatialCellSet used to determine PSF matching kernel """ if not candidateList: raise RuntimeError("Candidate list must be populated by makeCandidateList") results = self.matchMaskedImages( templateMaskedImage=templateMaskedImage, scienceMaskedImage=scienceMaskedImage, candidateList=candidateList, templateFwhmPix=templateFwhmPix, scienceFwhmPix=scienceFwhmPix, ) subtractedMaskedImage = afwImage.MaskedImageF(scienceMaskedImage, True) subtractedMaskedImage -= results.matchedImage subtractedMaskedImage -= results.backgroundModel results.subtractedMaskedImage = subtractedMaskedImage import lsstDebug display = lsstDebug.Info(__name__).display displayDiffIm = lsstDebug.Info(__name__).displayDiffIm maskTransparency = lsstDebug.Info(__name__).maskTransparency if not maskTransparency: maskTransparency = 0 if display: afwDisplay.setDefaultMaskTransparency(maskTransparency) if display and displayDiffIm: disp = afwDisplay.Display(frame=lsstDebug.frame) disp.mtv(subtractedMaskedImage, title="Subtracted masked image") lsstDebug.frame += 1 return results
def run(display=False): exposure = loadData() schema = afwTable.SourceTable.makeMinimalSchema() # # Create the detection task # config = SourceDetectionTask.ConfigClass() config.thresholdPolarity = "both" config.background.isNanSafe = True config.thresholdValue = 3 detectionTask = SourceDetectionTask(config=config, schema=schema) # # And the measurement Task # config = SingleFrameMeasurementTask.ConfigClass() config.algorithms.names = [ "base_SdssCentroid", "base_SdssShape", "base_CircularApertureFlux" ] config.algorithms["base_CircularApertureFlux"].radii = [ 1, 2, 4, 8, 12, 16 ] # pixels config.slots.gaussianFlux = None config.slots.modelFlux = None config.slots.psfFlux = None algMetadata = dafBase.PropertyList() measureTask = SingleFrameMeasurementTask(schema, algMetadata=algMetadata, config=config) radii = algMetadata.getArray("base_CircularApertureFlux_radii") # # Create the output table # tab = afwTable.SourceTable.make(schema) # # Process the data # result = detectionTask.run(tab, exposure) sources = result.sources print("Found %d sources (%d +ve, %d -ve)" % (len(sources), result.fpSets.numPos, result.fpSets.numNeg)) measureTask.run(sources, exposure) if display: # display image (see also --debug argparse option) afwDisplay.setDefaultMaskTransparency(75) frame = 1 disp = afwDisplay.Display(frame=frame) disp.mtv(exposure) with disp.Buffering(): for s in sources: xy = s.getCentroid() disp.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) disp.dot(s.getShape(), *xy, ctype=afwDisplay.RED) for radius in radii: disp.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
def selectSources(self, sourceCat, matches=None, exposure=None): """Return a selection of psf-like objects. Parameters ---------- sourceCat : `lsst.afw.table.SourceCatalog` Catalog of sources to select from. This catalog must be contiguous in memory. matches : `list` of `lsst.afw.table.ReferenceMatch` or None Ignored by this source selector. exposure : `lsst.afw.image.Exposure` or None The exposure the catalog was built from; used for debug display. Return ------ struct : `lsst.pipe.base.Struct` The struct contains the following data: - selected : `numpy.ndarray` of `bool`` Boolean array of sources that were selected, same length as sourceCat. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = display and \ lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells plotFwhmHistogram = display and plt and \ lsstDebug.Info(__name__).plotFwhmHistogram # Plot histogram of FWHM plotFlags = display and plt and \ lsstDebug.Info(__name__).plotFlags # Plot the sources coloured by their flags plotRejection = display and plt and \ lsstDebug.Info(__name__).plotRejection # Plot why sources are rejected afwDisplay.setDefaultMaskTransparency(75) fluxName = self.config.fluxName fluxErrName = self.config.fluxErrName minFwhm = self.config.minFwhm maxFwhm = self.config.maxFwhm maxFwhmVariability = self.config.maxFwhmVariability maxbad = self.config.maxbad maxbadflag = self.config.maxbadflag maxellip = self.config.maxellip minsn = self.config.minsn maxelong = (maxellip + 1.0)/(1.0 - maxellip) if maxellip < 1.0 else 100 # Unpack the catalogue shape = sourceCat.getShapeDefinition() ixx = sourceCat.get("%s.xx" % shape) iyy = sourceCat.get("%s.yy" % shape) fwhm = 2*np.sqrt(2*np.log(2))*np.sqrt(0.5*(ixx + iyy)) elong = 0.5*(ixx - iyy)/(ixx + iyy) flux = sourceCat.get(fluxName) fluxErr = sourceCat.get(fluxErrName) sn = flux/np.where(fluxErr > 0, fluxErr, 1) sn[fluxErr <= 0] = -psfexLib.BIG flags = 0x0 for i, f in enumerate(self.config.badFlags): flags = np.bitwise_or(flags, np.where(sourceCat.get(f), 1 << i, 0)) # # Estimate the acceptable range of source widths # good = np.logical_and(sn > minsn, np.logical_not(flags)) good = np.logical_and(good, elong < maxelong) good = np.logical_and(good, fwhm >= minFwhm) good = np.logical_and(good, fwhm < maxFwhm) fwhmMode, fwhmMin, fwhmMax = compute_fwhmrange(fwhm[good], maxFwhmVariability, minFwhm, maxFwhm, plot=dict(fwhmHistogram=plotFwhmHistogram)) # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Here's select_candidates # # ---- Apply some selection over flags, fluxes... bad = (flags != 0) # set.setBadFlags(int(sum(bad))) if plotRejection: selectionVectors = [] selectionVectors.append((bad, "flags %d" % sum(bad))) dbad = sn < minsn # set.setBadSN(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "S/N %d" % sum(dbad))) dbad = fwhm < fwhmMin # set.setBadFrmin(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "fwhmMin %d" % sum(dbad))) dbad = fwhm > fwhmMax # set.setBadFrmax(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "fwhmMax %d" % sum(dbad))) dbad = elong > maxelong # set.setBadElong(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "elong %d" % sum(dbad))) # -- ... and check the integrity of the sample if maxbadflag: nbad = np.array([(v <= -psfexLib.BIG).sum() for v in vignet]) dbad = nbad > maxbad # set.setBadPix(int(sum(dbad))) bad = np.logical_or(bad, dbad) if plotRejection: selectionVectors.append((dbad, "badpix %d" % sum(dbad))) good = np.logical_not(bad) # # We know enough to plot, if so requested # frame = 0 if displayExposure: mi = exposure.getMaskedImage() disp = afwDisplay.Display(frame=frame) disp.mtv(mi, title="PSF candidates") with disp.Buffering(): for i, source in enumerate(sourceCat): if good[i]: ctype = afwDisplay.GREEN # star candidate else: ctype = afwDisplay.RED # not star disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) if plotFlags or plotRejection: imag = -2.5*np.log10(flux) plt.clf() alpha = 0.5 if plotFlags: isSet = np.where(flags == 0x0)[0] plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label="good") for i, f in enumerate(self.config.badFlags): mask = 1 << i isSet = np.where(np.bitwise_and(flags, mask))[0] if isSet.any(): if np.isfinite(imag[isSet] + fwhm[isSet]).any(): label = re.sub(r"\_flag", "", re.sub(r"^base\_", "", re.sub(r"^.*base\_PixelFlags\_flag\_", "", f))) plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label=label) else: for bad, label in selectionVectors: plt.plot(imag[bad], fwhm[bad], 'o', alpha=alpha, label=label) plt.plot(imag[good], fwhm[good], 'o', color="black", label="selected") [plt.axhline(_, color='red') for _ in [fwhmMin, fwhmMax]] plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1])) plt.ylim(fwhm[np.where(np.isfinite(fwhm + imag))].min(), 2*fwhmMax) plt.legend(loc=2) plt.xlabel("Instrumental %s Magnitude" % fluxName.split(".")[-1].title()) plt.ylabel("fwhm") title = "PSFEX Star Selection" plt.title("%s %d selected" % (title, sum(good))) if displayExposure: global eventHandler eventHandler = EventHandler(plt.axes(), imag, fwhm, sourceCat.getX(), sourceCat.getY(), frames=[frame]) if plotFlags or plotRejection: while True: try: reply = input("continue? [y[es] h(elp) p(db) q(uit)] ").strip() except EOFError: reply = "y" if not reply: reply = "y" if reply[0] == "h": print("""\ At this prompt, you can continue with almost any key; 'p' enters pdb, 'q' returns to the shell, and 'h' prints this text """, end=' ') if displayExposure: print(""" If you put the cursor on a point in the matplotlib scatter plot and hit 'p' you'll see it in ds9.""") elif reply[0] == "p": import pdb pdb.set_trace() elif reply[0] == 'q': sys.exit(1) else: break return Struct(selected=good)
def _buildCellSet(self, exposure, referencePsfModel): """Build a SpatialCellSet for use with the solve method Parameters ---------- exposure : `lsst.afw.image.Exposure` The science exposure that will be convolved; must contain a Psf referencePsfModel : `lsst.afw.detection.Psf` Psf model to match to Returns ------- result : `struct` - ``kernelCellSet`` : a SpatialCellSet to be used by self._solve - ``referencePsfModel`` : Validated and/or modified reference model used to populate the SpatialCellSet Notes ----- If the reference Psf model and science Psf model have different dimensions, adjust the referencePsfModel (the model to which the exposure PSF will be matched) to match that of the science Psf. If the science Psf dimensions vary across the image, as is common with a WarpedPsf, either pad or clip (depending on config.padPsf) the dimensions to be constant. """ sizeCellX = self.kConfig.sizeCellX sizeCellY = self.kConfig.sizeCellY scienceBBox = exposure.getBBox() # Extend for proper spatial matching kernel all the way to edge, especially for narrow strips scienceBBox.grow(geom.Extent2I(sizeCellX, sizeCellY)) sciencePsfModel = exposure.getPsf() dimenR = referencePsfModel.getLocalKernel().getDimensions() psfWidth, psfHeight = dimenR regionSizeX, regionSizeY = scienceBBox.getDimensions() scienceX0, scienceY0 = scienceBBox.getMin() kernelCellSet = afwMath.SpatialCellSet(geom.Box2I(scienceBBox), sizeCellX, sizeCellY) nCellX = regionSizeX//sizeCellX nCellY = regionSizeY//sizeCellY if nCellX == 0 or nCellY == 0: raise ValueError("Exposure dimensions=%s and sizeCell=(%s, %s). Insufficient area to match" % (scienceBBox.getDimensions(), sizeCellX, sizeCellY)) # Survey the PSF dimensions of the Spatial Cell Set # to identify the minimum enclosed or maximum bounding square BBox. widthList = [] heightList = [] for row in range(nCellY): posY = sizeCellY*row + sizeCellY//2 + scienceY0 for col in range(nCellX): posX = sizeCellX*col + sizeCellX//2 + scienceX0 widthS, heightS = sciencePsfModel.computeBBox(geom.Point2D(posX, posY)).getDimensions() widthList.append(widthS) heightList.append(heightS) psfSize = max(max(heightList), max(widthList)) if self.config.doAutoPadPsf: minPsfSize = nextOddInteger(self.kConfig.kernelSize*self.config.autoPadPsfTo) paddingPix = max(0, minPsfSize - psfSize) else: if self.config.padPsfBy % 2 != 0: raise ValueError("Config padPsfBy (%i pixels) must be even number." % self.config.padPsfBy) paddingPix = self.config.padPsfBy if paddingPix > 0: self.log.info("Padding Science PSF from (%s, %s) to (%s, %s) pixels" % (psfSize, psfSize, paddingPix + psfSize, paddingPix + psfSize)) psfSize += paddingPix # Check that PSF is larger than the matching kernel maxKernelSize = psfSize - 1 if maxKernelSize % 2 == 0: maxKernelSize -= 1 if self.kConfig.kernelSize > maxKernelSize: message = """ Kernel size (%d) too big to match Psfs of size %d. Please reconfigure by setting one of the following: 1) kernel size to <= %d 2) doAutoPadPsf=True 3) padPsfBy to >= %s """ % (self.kConfig.kernelSize, psfSize, maxKernelSize, self.kConfig.kernelSize - maxKernelSize) raise ValueError(message) dimenS = geom.Extent2I(psfSize, psfSize) if (dimenR != dimenS): try: referencePsfModel = referencePsfModel.resized(psfSize, psfSize) self.log.info("Adjusted dimensions of reference PSF model from %s to %s" % (dimenR, dimenS)) except Exception as e: self.log.warn("Zero padding or clipping the reference PSF model of type %s and dimensions %s" " to the science Psf dimensions %s because: %s", referencePsfModel.__class__.__name__, dimenR, dimenS, e) dimenR = dimenS ps = pexConfig.makePropertySet(self.kConfig) for row in range(nCellY): # place at center of cell posY = sizeCellY*row + sizeCellY//2 + scienceY0 for col in range(nCellX): # place at center of cell posX = sizeCellX*col + sizeCellX//2 + scienceX0 log.log("TRACE4." + self.log.getName(), log.DEBUG, "Creating Psf candidate at %.1f %.1f", posX, posY) # reference kernel image, at location of science subimage referenceMI = self._makePsfMaskedImage(referencePsfModel, posX, posY, dimensions=dimenR) # kernel image we are going to convolve scienceMI = self._makePsfMaskedImage(sciencePsfModel, posX, posY, dimensions=dimenR) # The image to convolve is the science image, to the reference Psf. kc = diffimLib.makeKernelCandidate(posX, posY, scienceMI, referenceMI, ps) kernelCellSet.insertCandidate(kc) import lsstDebug display = lsstDebug.Info(__name__).display displaySpatialCells = lsstDebug.Info(__name__).displaySpatialCells maskTransparency = lsstDebug.Info(__name__).maskTransparency if not maskTransparency: maskTransparency = 0 if display: afwDisplay.setDefaultMaskTransparency(maskTransparency) if display and displaySpatialCells: dituils.showKernelSpatialCells(exposure.getMaskedImage(), kernelCellSet, symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=4, frame=lsstDebug.frame, title="Image to be convolved") lsstDebug.frame += 1 return pipeBase.Struct(kernelCellSet=kernelCellSet, referencePsfModel=referencePsfModel, )
import numpy as np import lsst.utils.tests import lsst.afw.math as afwMath import lsst.afw.image as afwImage import lsst.afw.image.utils as afwImageUtils from lsst.ip.isr.fringe import FringeTask try: display except NameError: display = False else: import lsst.afw.display as afwDisplay afwDisplay.setDefaultMaskTransparency(75) class FringeDataRef(object): """Quacks like a ButlerDataRef, so we can provide an in-memory fringe frame. """ def __init__(self, fringe): self.fringe = fringe self.dataId = {'test': True} def get(self, name="fringe", immediate=False): if name == "fringe": return self.fringe if name == "ccdExposureId": return 1000
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """Determine a PSFEX PSF model for an exposure given a list of PSF candidates. Parameters ---------- exposure: `lsst.afw.image.Exposure` Exposure containing the PSF candidates. psfCandidateList: iterable of `lsst.meas.algorithms.PsfCandidate` Sequence of PSF candidates typically obtained by detecting sources and then running them through a star selector. metadata: metadata, optional A home for interesting tidbits of information. flagKey: `lsst.afw.table.Key`, optional Schema key used to mark sources actually used in PSF determination. Returns ------- psf: `lsst.meas.extensions.psfex.PsfexPsf` The determined PSF. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = display and \ lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfComponents = display and \ lsstDebug.Info(__name__).displayPsfComponents # show the basis functions showBadCandidates = display and \ lsstDebug.Info(__name__).showBadCandidates # Include bad candidates (meaningless, methinks) displayResiduals = display and \ lsstDebug.Info(__name__).displayResiduals # show residuals displayPsfMosaic = display and \ lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals afwDisplay.setDefaultMaskTransparency(75) # Normalise residuals by object amplitude mi = exposure.getMaskedImage() nCand = len(psfCandidateList) if nCand == 0: raise RuntimeError("No PSF candidates supplied.") # # How big should our PSF models be? # if display: # only needed for debug plots # construct and populate a spatial cell set bbox = mi.getBBox(afwImage.PARENT) psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) else: psfCellSet = None sizes = np.empty(nCand) for i, psfCandidate in enumerate(psfCandidateList): try: if psfCellSet: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) rmsSize = quad.getTraceRadius() sizes[i] = rmsSize if self.config.kernelSize >= 15: self.log.warn("NOT scaling kernelSize by stellar quadrupole moment, but using absolute value") actualKernelSize = int(self.config.kernelSize) else: actualKernelSize = 2 * int(self.config.kernelSize * np.sqrt(np.median(sizes)) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: rms = np.median(sizes) print("Median PSF RMS size=%.2f pixels (\"FWHM\"=%.2f)" % (rms, 2*np.sqrt(2*np.log(2))*rms)) # If we manually set the resolution then we need the size in pixel units pixKernelSize = actualKernelSize if self.config.samplingSize > 0: pixKernelSize = int(actualKernelSize*self.config.samplingSize) if pixKernelSize % 2 == 0: pixKernelSize += 1 self.log.trace("Psfex Kernel size=%.2f, Image Kernel Size=%.2f", actualKernelSize, pixKernelSize) psfCandidateList[0].setHeight(pixKernelSize) psfCandidateList[0].setWidth(pixKernelSize) # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- BEGIN PSFEX # # Insert the good candidates into the set # defaultsFile = os.path.join(os.environ["MEAS_EXTENSIONS_PSFEX_DIR"], "config", "default-lsst.psfex") args_md = dafBase.PropertySet() args_md.set("BASIS_TYPE", str(self.config.psfexBasis)) args_md.set("PSFVAR_DEGREES", str(self.config.spatialOrder)) args_md.set("PSF_SIZE", str(actualKernelSize)) args_md.set("PSF_SAMPLING", str(self.config.samplingSize)) prefs = psfex.Prefs(defaultsFile, args_md) prefs.setCommandLine([]) prefs.addCatalog("psfexPsfDeterminer") prefs.use() principalComponentExclusionFlag = bool(bool(psfex.Context.REMOVEHIDDEN) if False else psfex.Context.KEEPHIDDEN) context = psfex.Context(prefs.getContextName(), prefs.getContextGroup(), prefs.getGroupDeg(), principalComponentExclusionFlag) set = psfex.Set(context) set.setVigSize(pixKernelSize, pixKernelSize) set.setFwhm(2*np.sqrt(2*np.log(2))*np.median(sizes)) set.setRecentroid(self.config.recentroid) catindex, ext = 0, 0 backnoise2 = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue() ccd = exposure.getDetector() if ccd: gain = np.mean(np.array([a.getGain() for a in ccd])) else: gain = 1.0 self.log.warn("Setting gain to %g" % (gain,)) contextvalp = [] for i, key in enumerate(context.getName()): if context.getPcflag(i): contextvalp.append(pcval[pc]) pc += 1 elif key[0] == ':': try: contextvalp.append(exposure.getMetadata().getScalar(key[1:])) except KeyError: raise RuntimeError("*Error*: %s parameter not found in the header of %s" % (key[1:], prefs.getContextName())) else: try: contextvalp.append(np.array([psfCandidateList[_].getSource().get(key) for _ in range(nCand)])) except KeyError: raise RuntimeError("*Error*: %s parameter not found" % (key,)) set.setContextname(i, key) if display: frame = 0 if displayExposure: disp = afwDisplay.Display(frame=frame) disp.mtv(exposure, title="psf determination") badBits = mi.getMask().getPlaneBitMask(self.config.badMaskBits) fluxName = prefs.getPhotfluxRkey() fluxFlagName = "base_" + fluxName + "_flag" xpos, ypos = [], [] for i, psfCandidate in enumerate(psfCandidateList): source = psfCandidate.getSource() xc, yc = source.getX(), source.getY() try: int(xc), int(yc) except ValueError: continue try: pstamp = psfCandidate.getMaskedImage().clone() except Exception: continue if fluxFlagName in source.schema and source.get(fluxFlagName): continue flux = source.get(fluxName) if flux < 0 or np.isnan(flux): continue # From this point, we're configuring the "sample" (PSFEx's version of a PSF candidate). # Having created the sample, we must proceed to configure it, and then fini (finalize), # or it will be malformed. try: sample = set.newSample() sample.setCatindex(catindex) sample.setExtindex(ext) sample.setObjindex(i) imArray = pstamp.getImage().getArray() imArray[np.where(np.bitwise_and(pstamp.getMask().getArray(), badBits))] = \ -2*psfex.BIG sample.setVig(imArray) sample.setNorm(flux) sample.setBacknoise2(backnoise2) sample.setGain(gain) sample.setX(xc) sample.setY(yc) sample.setFluxrad(sizes[i]) for j in range(set.getNcontext()): sample.setContext(j, float(contextvalp[j][i])) except Exception as e: self.log.debug("Exception when processing sample at (%f,%f): %s", xc, yc, e) continue else: set.finiSample(sample) xpos.append(xc) # for QA ypos.append(yc) if displayExposure: with disp.Buffering(): disp.dot("o", xc, yc, ctype=afwDisplay.CYAN, size=4) if set.getNsample() == 0: raise RuntimeError("No good PSF candidates to pass to PSFEx") # ---- Update min and max and then the scaling for i in range(set.getNcontext()): cmin = contextvalp[i].min() cmax = contextvalp[i].max() set.setContextScale(i, cmax - cmin) set.setContextOffset(i, (cmin + cmax)/2.0) # Don't waste memory! set.trimMemory() # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- END PSFEX # # Do a PSFEX decomposition of those PSF candidates # fields = [] field = psfex.Field("Unknown") field.addExt(exposure.getWcs(), exposure.getWidth(), exposure.getHeight(), set.getNsample()) field.finalize() fields.append(field) sets = [] sets.append(set) psfex.makeit(fields, sets) psfs = field.getPsfs() # Flag which objects were actually used in psfex by good_indices = [] for i in range(sets[0].getNsample()): index = sets[0].getSample(i).getObjindex() if index > -1: good_indices.append(index) if flagKey is not None: for i, psfCandidate in enumerate(psfCandidateList): source = psfCandidate.getSource() if i in good_indices: source.set(flagKey, True) xpos = np.array(xpos) ypos = np.array(ypos) numGoodStars = len(good_indices) avgX, avgY = np.mean(xpos), np.mean(ypos) psf = psfex.PsfexPsf(psfs[0], afwGeom.Point2D(avgX, avgY)) if False and (displayResiduals or displayPsfMosaic): ext = 0 frame = 1 diagnostics = True catDir = "." title = "psfexPsfDeterminer" psfex.psfex.showPsf(psfs, set, ext, [(exposure.getWcs(), exposure.getWidth(), exposure.getHeight())], nspot=3, trim=5, frame=frame, diagnostics=diagnostics, outDir=catDir, title=title) # # Display code for debugging # if display: assert psfCellSet is not None if displayExposure: maUtils.showPsfSpatialCells(exposure, psfCellSet, showChi2=True, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=8, display=disp) if displayResiduals: disp4 = afwDisplay.Display(frame=4) maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) maUtils.showPsf(psf, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) maUtils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp.scale('linear', 0, 1) # # Generate some QA information # # Count PSF stars # if metadata is not None: metadata.set("spatialFitChi2", np.nan) metadata.set("numAvailStars", nCand) metadata.set("numGoodStars", numGoodStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) return psf, psfCellSet