def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128, matchKernelAmplitudes=False, keepPlots=True): """Plot the PSF spatial model.""" if plt is None: print >> sys.stderr, "Unable to import matplotlib" return noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) candPos = list() candFits = list() badPos = list() badFits = list() candAmps = list() badAmps = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) if not showBadCandidates and cand.isBad(): continue candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage() except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() targetFits = badFits if cand.isBad() else candFits targetPos = badPos if cand.isBad() else candPos targetAmps = badAmps if cand.isBad() else candAmps targetFits.append([x / amp for x in params]) targetPos.append(candCenter) targetAmps.append(amp)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """!Determine a PCA PSF model for an exposure given a list of PSF candidates \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure) \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector \param[in,out] metadata a home for interesting tidbits of information \param[in] flagKey schema key used to mark sources actually used in PSF determination \return a list of - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info( __name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info( __name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info( __name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info( __name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info( __name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) # match Kernel amplitudes for spatial plots matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes # Keep matplotlib alive post mortem keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots displayPsfSpatialModel = lsstDebug.Info( __name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info( __name__).showBadCandidates # Include bad candidates # Normalize residuals by object amplitude normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals pause = lsstDebug.Info( __name__).pause # Prompt user after each iteration? if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.log.warn( "WARNING: NOT scaling kernelSize by stellar quadrupole moment " + "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead", self.config.kernelSize) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2 * int(self.config.kernelSize * math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print("Median size=%s" % (medSize, )) self.log.trace("Kernel size=%s", actualKernelSize) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [ ] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print("Removing %d blended Psf candidates" % len(blendedCandidates)) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW, size=4, frame=frame) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iterNum in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates # import lsst.afw.display.utils as displayUtils stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates ): # maybe include bad candidates cand = algorithmsLib.PsfCandidateF.cast(cand) try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append( (im, "%d%s" % (maUtils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception as e: continue if len(stamps) == 0: print( "WARNING: No PSF candidates to show; try setting showBadCandidates=True" ) else: mos = displayUtils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics( im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append( im, label, ds9.GREEN if status == afwMath.SpatialCellCandidate.GOOD else ds9.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED) mos.makeMosaic(frame=8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.PsfCandidateF.cast(cand) cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN ) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.log.debug("chi^2=%s; id=%s", cand.getChi2(), cand.getSource().getId()) for cand in awfulCandidates: if display: print("Removing bad candidate: id=%d, chi^2=%f" % \ (cand.getSource().getId(), cand.getChi2())) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.PsfCandidateF.cast(cand) rchi2 = cand.getChi2( ) # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = afwMath.cast_LinearCombinationKernel( psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.PsfCandidateF.cast(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception as e: continue fit = algorithmsLib.fitKernelParamsToImage( noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() predict = [ kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters()) ] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append( [a / amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:, k].mean() rms = residuals[:, k].std() elif False: # Using interquartile range sr = numpy.sort(residuals[:, k]) mean = sr[int(0.5*len(sr))] if len(sr) % 2 else \ 0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1]) rms = 0.74 * (sr[int(0.75 * len(sr))] - sr[int(0.25 * len(sr))]) else: stats = afwMath.makeStatistics( residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP) mean = stats.getValue(afwMath.MEANCLIP) rms = stats.getValue(afwMath.STDEVCLIP) rms = max( 1.0e-4, rms) # Don't trust RMS below this due to numerical issues if display: print("Mean for component %d is %f" % (k, mean)) print("RMS for component %d is %f" % (k, rms)) badCandidates = list() for i, cand in enumerate(candidates): if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject * rms: badCandidates.append(i) badCandidates.sort( key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates): cand = candidates[c] if display: print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \ (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k, residuals[badCandidates[i], k], self.config.spatialReject * rms)) cand.setStatus(afwMath.SpatialCellCandidate.BAD) # # Display results # if display and displayIterations: if displayExposure: if iterNum > 0: ds9.erase(frame=frame) maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", size=8, frame=frame, ctype=ds9.YELLOW, ctypeBad=ds9.RED, ctypeUnused=ds9.MAGENTA) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: maUtils.showPsfSpatialCells( exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", size=10, frame=frame, ctype=ds9.YELLOW, ctypeBad=ds9.RED) if displayResiduals: while True: try: maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = input( "Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:] else: reply = "" if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"): if reply == "c": pause = False elif reply == "h": print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \ "s[ave fileName] y[es]") continue elif reply == "p": import pdb pdb.set_trace() elif reply == "q": display = False elif reply == "Q": sys.exit(1) elif reply == "s": fileName = args.pop(0) if not fileName: print("Please provide a filename") continue print("Saving to %s" % fileName) maUtils.saveSpatialCellSet(psfCellSet, fileName=fileName) continue break else: print("Unrecognised response: %s" % reply, file=sys.stderr) if reply == "n": break # One last time, to take advantage of the last iteration psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # Display code for debugging # if display and reply != "n": if displayExposure: maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: maUtils.showPsfSpatialCells( exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=10, frame=frame) if displayResiduals: maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale("linear", 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) # # Generate some QA information # # Count PSF stars # numGoodStars = 0 numAvailStars = 0 avgX = 0.0 avgY = 0.0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # don't ignore BAD stars numAvailStars += 1 for cand in cell.begin(True): # do ignore BAD stars cand = algorithmsLib.PsfCandidateF.cast(cand) src = cand.getSource() if flagKey is not None: src.set(flagKey, True) avgX += src.getX() avgY += src.getY() numGoodStars += 1 avgX /= numGoodStars avgY /= numGoodStars if metadata is not None: metadata.set("spatialFitChi2", fitChi2) metadata.set("numGoodStars", numGoodStars) metadata.set("numAvailStars", numAvailStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psf = algorithmsLib.PcaPsf(psf.getKernel(), afwGeom.Point2D(avgX, avgY)) return psf, psfCellSet
noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters())] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append([a / amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:,k].mean()
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128, matchKernelAmplitudes=False, keepPlots=True): """Plot the PSF spatial model.""" if not plt: print >> sys.stderr, "Unable to import matplotlib" return noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) candPos = list() candFits = list() badPos = list() badFits = list() candAmps = list() badAmps = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) if not showBadCandidates and cand.isBad(): continue candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage() except Exception: continue fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() targetFits = badFits if cand.isBad() else candFits targetPos = badPos if cand.isBad() else candPos targetAmps = badAmps if cand.isBad() else candAmps targetFits.append([x / amp for x in params]) targetPos.append(candCenter) targetAmps.append(amp) numCandidates = len(candFits) numBasisFuncs = noSpatialKernel.getNBasisKernels() xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0() yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0() zGood = numpy.array(candFits) ampGood = numpy.array(candAmps) xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0() yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0() zBad = numpy.array(badFits) ampBad = numpy.array(badAmps) numBad = len(badPos) xRange = numpy.linspace(0, exposure.getWidth(), num=numSample) yRange = numpy.linspace(0, exposure.getHeight(), num=numSample) kernel = psf.getKernel() nKernelComponents = kernel.getNKernelParameters() # # Figure out how many panels we'll need # nPanelX = int(math.sqrt(nKernelComponents)) nPanelY = nKernelComponents//nPanelX while nPanelY*nPanelX < nKernelComponents: nPanelX += 1 fig = plt.figure(1) fig.clf() try: fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word except: # protect against API changes pass # # Generator for axes arranged in panels # subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04) for k in range(nKernelComponents): func = kernel.getSpatialFunction(k) dfGood = zGood[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos]) yMin = dfGood.min() yMax = dfGood.max() if numBad > 0: dfBad = zBad[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos]) yMin = min([yMin, dfBad.min()]) yMax = max([yMax, dfBad.max()]) yMin -= 0.05 * (yMax - yMin) yMax += 0.05 * (yMax - yMin) yMin = -0.01 yMax = 0.01 fRange = numpy.ndarray((len(xRange), len(yRange))) for j, yVal in enumerate(yRange): for i, xVal in enumerate(xRange): fRange[j][i] = func(xVal, yVal) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() ax.set_autoscale_on(False) ax.set_xbound(lower=0, upper=exposure.getHeight()) ax.set_ybound(lower=yMin, upper=yMax) ax.plot(yGood, dfGood, 'b+') if numBad > 0: ax.plot(yBad, dfBad, 'r+') ax.axhline(0.0) ax.set_title('Residuals(y)') #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() if matchKernelAmplitudes and k == 0: vmin = 0.0 vmax = 1.1 else: vmin = fRange.min() vmax = fRange.max() norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax) im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm, extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1]) ax.set_title('Spatial poly') plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax]) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() ax.set_autoscale_on(False) ax.set_xbound(lower=0, upper=exposure.getWidth()) ax.set_ybound(lower=yMin, upper=yMax) ax.plot(xGood, dfGood, 'b+') if numBad > 0: ax.plot(xBad, dfBad, 'r+') ax.axhline(0.0) ax.set_title('K%d Residuals(x)' % k) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() if False: ax.scatter(xGood, yGood, c=dfGood, marker='o') ax.scatter(xBad, yBad, c=dfBad, marker='x') ax.set_xbound(lower=0, upper=exposure.getWidth()) ax.set_ybound(lower=0, upper=exposure.getHeight()) ax.set_title('Spatial residuals') plt.colorbar(im, orientation='horizontal') else: calib = exposure.getCalib() if calib.getFluxMag0()[0] <= 0: calib = type(calib)() calib.setFluxMag0(1.0) with CalibNoThrow(): ax.plot(calib.getMagnitude(candAmps), zGood[:,k], 'b+') if numBad > 0: ax.plot(calib.getMagnitude(badAmps), zBad[:,k], 'r+') ax.set_title('Flux variation') fig.show() global keptPlots if keepPlots and not keptPlots: # Keep plots open when done def show(): print "%s: Please close plots when done." % __name__ try: plt.show() except: pass print "Plots closed, exiting..." import atexit atexit.register(show) keptPlots = True
class PcaPsfDeterminer(object): """! A measurePsfTask psf estimator """ ConfigClass = PcaPsfDeterminerConfig def __init__(self, config): """!Construct a PCA PSF Fitter \param[in] config instance of PcaPsfDeterminerConfig """ self.config = config # N.b. name of component is meas.algorithms.psfDeterminer so you can turn on psf debugging # independent of which determiner is active self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer") self.warnLog = pexLog.Log(pexLog.getDefaultLog(), "meas.algorithms.psfDeterminer") def _fitPsf(self, exposure, psfCellSet, kernelSize, nEigenComponents): algorithmsLib.PsfCandidateF.setPixelThreshold( self.config.pixelThreshold) algorithmsLib.PsfCandidateF.setMaskBlends(self.config.doMaskBlends) # # Loop trying to use nEigenComponents, but allowing smaller numbers if necessary # for nEigen in range(nEigenComponents, 0, -1): # Determine KL components try: kernel, eigenValues = algorithmsLib.createKernelFromPsfCandidates( psfCellSet, exposure.getDimensions(), exposure.getXY0(), nEigen, self.config.spatialOrder, kernelSize, self.config.nStarPerCell, bool(self.config.constantWeight)) break # OK, we can get nEigen components except pexExceptions.LengthError as e: if nEigen == 1: # can't go any lower raise IndexError("No viable PSF candidates survive") self.warnLog.log( pexLog.Log.WARN, "%s: reducing number of eigen components" % e.what()) # # We got our eigen decomposition so let's use it # # Express eigenValues in units of reduced chi^2 per star size = kernelSize + 2 * self.config.borderWidth nu = size * size - 1 # number of degrees of freedom/star for chi^2 eigenValues = [ l / float( algorithmsLib.countPsfCandidates( psfCellSet, self.config.nStarPerCell) * nu) for l in eigenValues ] # Fit spatial model status, chi2 = algorithmsLib.fitSpatialKernelFromPsfCandidates( kernel, psfCellSet, bool(self.config.nonLinearSpatialFit), self.config.nStarPerCellSpatialFit, self.config.tolerance, self.config.lam) psf = algorithmsLib.PcaPsf(kernel) return psf, eigenValues, nEigen, chi2 def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """!Determine a PCA PSF model for an exposure given a list of PSF candidates \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure) \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector \param[in,out] metadata a home for interesting tidbits of information \param[in] flagKey schema key used to mark sources actually used in PSF determination \return a list of - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info( __name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info( __name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info( __name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info( __name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info( __name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) matchKernelAmplitudes = lsstDebug.Info( __name__).matchKernelAmplitudes # match Kernel amplitudes # for spatial plots keepMatplotlibPlots = lsstDebug.Info( __name__).keepMatplotlibPlots # Keep matplotlib alive # post mortem displayPsfSpatialModel = lsstDebug.Info( __name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info( __name__).showBadCandidates # Include bad candidates normalizeResiduals = lsstDebug.Info( __name__).normalizeResiduals # Normalise residuals by # object amplitude pause = lsstDebug.Info( __name__).pause # Prompt user after each iteration? if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception, e: self.debugLog.debug( 2, "Skipping PSF candidate %d of %d: %s" % (i, len(psfCandidateList), e)) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.debugLog.debug(1, \ "WARNING: NOT scaling kernelSize by stellar quadrupole moment " + "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead" \ % (self.config.kernelSize,) ) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2 * int(self.config.kernelSize * math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print "Median size=%s" % (medSize, ) self.debugLog.debug(3, "Kernel size=%s" % (actualKernelSize, )) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [ ] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print "Removing %d blended Psf candidates" % len( blendedCandidates) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW, size=4, frame=frame) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iter in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates # import lsst.afw.display.utils as displayUtils stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates ): # maybe include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append( (im, "%d%s" % (maUtils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception, e: continue if len(stamps) == 0: print "WARNING: No PSF candidates to show; try setting showBadCandidates=True" else: mos = displayUtils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics( im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append( im, label, ds9.GREEN if status == afwMath.SpatialCellCandidate.GOOD else ds9.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED) mos.makeMosaic(frame=8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN ) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.debugLog.debug( 2, "chi^2=%s; id=%s" % (cand.getChi2(), cand.getSource().getId())) for cand in awfulCandidates: if display: print "Removing bad candidate: id=%d, chi^2=%f" % \ (cand.getSource().getId(), cand.getChi2()) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) rchi2 = cand.getChi2( ) # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = int( len(badCandidates) * (iter + 1) / self.config.nIterForPsf + 0.5) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print "Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = afwMath.cast_LinearCombinationKernel( psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage( noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() predict = [ kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters()) ] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append( [a / amp - p for a, p in zip(params, predict)]) candidates.append(cand)
cand = algorithmsLib.cast_PsfCandidateF(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage( noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() predict = [ kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters()) ] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append( [a / amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals)