def setUp(self): self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(-20, -30), lsst.geom.Extent2I(240, 160)) self.dataset = measBaseTests.TestDataset(self.bbox) # first two sources are points self.pointCentroid1 = lsst.geom.Point2D(50.1, 49.8) self.pointCentroid2 = lsst.geom.Point2D(-11.6, -1.7) self.dataset.addSource(instFlux=1E5, centroid=self.pointCentroid1) self.dataset.addSource(instFlux=2E5, centroid=self.pointCentroid2) # third source is extended self.extendedCentroid = lsst.geom.Point2D(149.9, 50.3) self.dataset.addSource(instFlux=1E5, centroid=self.extendedCentroid, shape=afwGeom.Quadrupole(8, 9, 3)) self.config = self.makeSingleFrameMeasurementConfig("base_SdssShape")
def dot(self, symb, c, r, size=2, ctype=None, origin=afwImage.PARENT, *args, **kwargs): """!Draw a symbol onto the specified DISPLAY frame at (col,row) = (c,r) [0-based coordinates] Possible values are: + Draw a + x Draw an x * Draw a * o Draw a circle @:Mxx,Mxy,Myy Draw an ellipse with moments (Mxx, Mxy, Myy) (argument size is ignored) An object derived from afwGeom.ellipses.BaseCore Draw the ellipse (argument size is ignored) Any other value is interpreted as a string to be drawn. Strings obey the fontFamily (which may be extended with other characteristics, e.g. "times bold italic". Text will be drawn rotated by textAngle (textAngle is ignored otherwise). N.b. objects derived from BaseCore include Axes and Quadrupole. """ if isinstance(symb, int): symb = "%d" % (symb) if origin == afwImage.PARENT and self._xy0 is not None: x0, y0 = self._xy0 r -= y0 c -= x0 if isinstance(symb, afwGeom.ellipses.BaseCore) or re.search( r"^@:", symb): try: mat = re.search(r"^@:([^,]+),([^,]+),([^,]+)", symb) except TypeError: pass else: if mat: mxx, mxy, myy = [float(_) for _ in mat.groups()] symb = afwGeom.Quadrupole(mxx, myy, mxy) symb = afwGeom.ellipses.Axes(symb) self._impl._dot(symb, c, r, size, ctype, **kwargs)
def testRejectBlends(self): """Test the PcaPsfDeterminerTask blend removal.""" """ We give it a single blended source, asking it to remove blends, and check that it barfs in the expected way. """ psfDeterminerClass = measAlg.psfDeterminerRegistry["pca"] config = psfDeterminerClass.ConfigClass() config.doRejectBlends = True psfDeterminer = psfDeterminerClass(config=config) schema = afwTable.SourceTable.makeMinimalSchema() # Use The single frame measurement task to populate the schema with standard keys measBase.SingleFrameMeasurementTask(schema) catalog = afwTable.SourceCatalog(schema) source = catalog.addNew() # Make the source blended, with necessary information to calculate pca spanShift = afwGeom.Point2I(54, 123) spans = afwGeom.SpanSet.fromShape(6, offset=spanShift) foot = afwDetection.Footprint(spans, self.exposure.getBBox()) foot.addPeak(45, 123, 6) foot.addPeak(47, 126, 5) source.setFootprint(foot) centerKey = afwTable.Point2DKey(source.schema['slot_Centroid']) shapeKey = afwTable.QuadrupoleKey(schema['slot_Shape']) source.set(centerKey, afwGeom.Point2D(46, 124)) source.set(shapeKey, afwGeom.Quadrupole(1.1, 2.2, 1)) candidates = [measAlg.makePsfCandidate(source, self.exposure)] metadata = dafBase.PropertyList() with self.assertRaises(RuntimeError) as cm: psfDeterminer.determinePsf(self.exposure, candidates, metadata) self.assertEqual(str(cm.exception), "All PSF candidates removed as blends")
def dot(self, symb, c, r, size=2, ctype=None, origin=afwImage.PARENT, *args, **kwargs): """Draw a symbol onto the specified display frame Parameters ---------- symb Possible values are: ``"+"`` Draw a + ``"x"`` Draw an x ``"*"`` Draw a * ``"o"`` Draw a circle ``"@:Mxx,Mxy,Myy"`` Draw an ellipse with moments (Mxx, Mxy, Myy) (argument size is ignored) `lsst.afw.geom.ellipses.BaseCore` Draw the ellipse (argument size is ignored). N.b. objects derived from `~lsst.afw.geom.ellipses.BaseCore` include `~lsst.afw.geom.ellipses.Axes` and `~lsst.afw.geom.ellipses.Quadrupole`. Any other value Interpreted as a string to be drawn. c, r The column and row where the symbol is drawn [0-based coordinates] size : `int` Size of symbol, in pixels ctype : `str` The desired color, either e.g. `lsst.afw.display.RED` or a color name known to X11 origin : `lsst.afw.image.ImageOrigin` Coordinate system for the given positions. *args Extra arguments to backend **kwargs Extra keyword arguments to backend """ if isinstance(symb, int): symb = f"{symb:d}" if origin == afwImage.PARENT and self._xy0 is not None: x0, y0 = self._xy0 r -= y0 c -= x0 if isinstance(symb, afwGeom.ellipses.BaseCore) or re.search( r"^@:", symb): try: mat = re.search(r"^@:([^,]+),([^,]+),([^,]+)", symb) except TypeError: pass else: if mat: mxx, mxy, myy = [float(_) for _ in mat.groups()] symb = afwGeom.Quadrupole(mxx, myy, mxy) symb = afwGeom.ellipses.Axes(symb) self._impl._dot(symb, c, r, size, ctype, **kwargs)
def plantSources(x0, y0, nx, ny, sky, nObj, wid, detector, useRandom=False): pixToTanPix = detector.getTransform(cameraGeom.PIXELS, cameraGeom.TAN_PIXELS) img0 = afwImage.ImageF(afwGeom.ExtentI(nx, ny)) img = afwImage.ImageF(afwGeom.ExtentI(nx, ny)) ixx0, iyy0, ixy0 = wid * wid, wid * wid, 0.0 edgeBuffer = 40.0 * wid flux = 1.0e4 nkx, nky = int(10 * wid) + 1, int(10 * wid) + 1 xhwid, yhwid = nkx // 2, nky // 2 nRow = int(math.sqrt(nObj)) xstep = (nx - 1 - 0.0 * edgeBuffer) // (nRow + 1) ystep = (ny - 1 - 0.0 * edgeBuffer) // (nRow + 1) if useRandom: nObj = nRow * nRow goodAdded0 = [] goodAdded = [] for i in range(nObj): # get our position if useRandom: xcen0, ycen0 = np.random.uniform(nx), np.random.uniform(ny) else: xcen0, ycen0 = xstep * ( (i % nRow) + 1), ystep * (int(i / nRow) + 1) ixcen0, iycen0 = int(xcen0), int(ycen0) # distort position and shape pTan = afwGeom.Point2D(xcen0, ycen0) p = pixToTanPix.applyInverse(pTan) linTransform = afwGeom.linearizeTransform(pixToTanPix, p).invert().getLinear() m = afwGeom.Quadrupole(ixx0, iyy0, ixy0) m.transform(linTransform) xcen, ycen = xcen0, ycen0 # p.getX(), p.getY() if (xcen < 1.0 * edgeBuffer or (nx - xcen) < 1.0 * edgeBuffer or ycen < 1.0 * edgeBuffer or (ny - ycen) < 1.0 * edgeBuffer): continue ixcen, iycen = int(xcen), int(ycen) ixx, iyy, ixy = m.getIxx(), m.getIyy(), m.getIxy() # plant the object tmp = 0.25 * (ixx - iyy)**2 + ixy**2 a2 = 0.5 * (ixx + iyy) + np.sqrt(tmp) b2 = 0.5 * (ixx + iyy) - np.sqrt(tmp) theta = 0.5 * np.arctan2(2.0 * ixy, ixx - iyy) a = np.sqrt(a2) b = np.sqrt(b2) c, s = math.cos(theta), math.sin(theta) good0, good = True, True for y in range(nky): iy = iycen + y - yhwid iy0 = iycen0 + y - yhwid for x in range(nkx): ix = ixcen + x - xhwid ix0 = ixcen0 + x - xhwid if ix >= 0 and ix < nx and iy >= 0 and iy < ny: dx, dy = ix - xcen, iy - ycen u = c * dx + s * dy v = -s * dx + c * dy I0 = flux / (2 * math.pi * a * b) val = I0 * math.exp(-0.5 * ((u / a)**2 + (v / b)**2)) if val < 0: val = 0 prevVal = img.get(ix, iy) img.set(ix, iy, val + prevVal) else: good = False if ix0 >= 0 and ix0 < nx and iy0 >= 0 and iy0 < ny: dx, dy = ix - xcen, iy - ycen I0 = flux / (2 * math.pi * wid * wid) val = I0 * math.exp(-0.5 * ((dx / wid)**2 + (dy / wid)**2)) if val < 0: val = 0 prevVal = img0.get(ix0, iy0) img0.set(ix0, iy0, val + prevVal) else: good0 = False if good0: goodAdded0.append([xcen, ycen]) if good: goodAdded.append([xcen, ycen]) # add sky and noise img += sky img0 += sky noise = afwImage.ImageF(afwGeom.ExtentI(nx, ny)) noise0 = afwImage.ImageF(afwGeom.ExtentI(nx, ny)) for i in range(nx): for j in range(ny): noise.set(i, j, np.random.poisson(img.get(i, j))) noise0.set(i, j, np.random.poisson(img0.get(i, j))) edgeWidth = int(0.5 * edgeBuffer) mask = afwImage.Mask(afwGeom.ExtentI(nx, ny)) left = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.ExtentI(edgeWidth, ny)) right = afwGeom.Box2I(afwGeom.Point2I(nx - edgeWidth, 0), afwGeom.ExtentI(edgeWidth, ny)) top = afwGeom.Box2I(afwGeom.Point2I(0, ny - edgeWidth), afwGeom.ExtentI(nx, edgeWidth)) bottom = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.ExtentI(nx, edgeWidth)) for pos in [left, right, top, bottom]: msk = afwImage.Mask(mask, pos, deep=False) msk.set(msk.getPlaneBitMask('EDGE')) expos = afwImage.makeExposure( afwImage.makeMaskedImage(noise, mask, afwImage.ImageF(noise, True))) expos0 = afwImage.makeExposure( afwImage.makeMaskedImage(noise0, mask, afwImage.ImageF(noise0, True))) im = expos.getMaskedImage().getImage() im0 = expos0.getMaskedImage().getImage() im -= sky im0 -= sky return expos, goodAdded, expos0, goodAdded0
def main(dataDir, visit, title="", outputTxtFileName=None, showFwhm=False, minFwhm=None, maxFwhm=None, correctDistortion=False, showEllipticity=False, ellipticityDirection=False, showNdataFwhm=False, showNdataEll=False, minNdata=None, maxNdata=None, gridPoints=30, verbose=False): butler = dafPersist.ButlerFactory(mapper=hscSim.HscSimMapper(root=dataDir)).create() camera = butler.get("camera") if not (showFwhm or showEllipticity or showNdataFwhm or showNdataEll or outputTxtFileName): showFwhm = True # # Get a dict of cameraGeom::Ccd indexed by serial number # ccds = {} for raft in camera: for ccd in raft: ccd.setTrimmed(True) ccds[ccd.getId().getSerial()] = ccd # # Read all the tableSeeingMap files, converting their (x, y) to focal # plane coordinates # xArr = [] yArr = [] ellArr = [] fwhmArr = [] paArr = [] aArr = [] bArr = [] e1Arr = [] e2Arr = [] elle1e2Arr = [] for tab in butler.subset("tableSeeingMap", visit=visit): # we could use tab.datasetExists() but it prints a rude message fileName = butler.get("tableSeeingMap_filename", **tab.dataId)[0] if not os.path.exists(fileName): continue with open(fileName) as fd: ccd = None for line in fd.readlines(): if re.search(r"^\s*#", line): continue fields = [float(_) for _ in line.split()] if ccd is None: ccd = ccds[int(fields[0])] x, y, fwhm, ell, pa, a, b = fields[1:8] x, y = ccd.getPositionFromPixel(geom.PointD(x, y)).getMm() xArr.append(x) yArr.append(y) ellArr.append(ell) fwhmArr.append(fwhm) paArr.append(pa) aArr.append(a) bArr.append(b) if len(fields) == 11: e1 = fields[8] e2 = fields[9] elle1e2 = fields[10] else: e1 = -9999. e2 = -9999. elle1e2 = -9999. e1Arr.append(e1) e2Arr.append(e2) elle1e2Arr.append(elle1e2) xArr = np.array(xArr) yArr = np.array(yArr) ellArr = np.array(ellArr) fwhmArr = np.array(fwhmArr)*0.168 # arcseconds paArr = np.radians(np.array(paArr)) aArr = np.array(aArr) bArr = np.array(bArr) e1Arr = np.array(e1Arr) e2Arr = np.array(e2Arr) elle1e2Arr = np.array(elle1e2Arr) if correctDistortion: import lsst.afw.geom.ellipses as afwEllipses dist = camera.getDistortion() for i in range(len(aArr)): axes = afwEllipses.Axes(aArr[i], bArr[i], paArr[i]) if False: # testing only! axes = afwEllipses.Axes(1.0, 1.0, np.arctan2(yArr[i], xArr[i])) quad = afwGeom.Quadrupole(axes) quad = quad.transform(dist.computeQuadrupoleTransform(geom.PointD(xArr[i], yArr[i]), False)) axes = afwEllipses.Axes(quad) aArr[i], bArr[i], paArr[i] = axes.getA(), axes.getB(), axes.getTheta() ellArr = 1 - bArr/aArr if len(xArr) == 0: gridPoints = 0 xs, ys = [], [] else: N = gridPoints*1j extent = [min(xArr), max(xArr), min(yArr), max(yArr)] xs, ys = np.mgrid[extent[0]:extent[1]:N, extent[2]:extent[3]:N] title = [title, ] title.append("\n#") if outputTxtFileName: f = open(outputTxtFileName, 'w') f.write("# %s visit %s\n" % (" ".join(title), visit)) for x, y, ell, fwhm, pa, a, b, e1, e2, elle1e2 \ in zip(xArr, yArr, ellArr, fwhmArr, paArr, aArr, bArr, e1Arr, e2Arr, elle1e2Arr): f.write('%f %f %f %f %f %f %f %f %f %f\n' % (x, y, ell, fwhm, pa, a, b, e1, e2, elle1e2)) if showFwhm: title.append("FWHM (arcsec)") if len(xs) > 0: fwhmResampled = griddata(xArr, yArr, fwhmArr, xs, ys) plt.imshow(fwhmResampled.T, extent=extent, vmin=minFwhm, vmax=maxFwhm, origin='lower') plt.colorbar() if outputTxtFileName: ndataGrids = getNumDataGrids(xArr, yArr, fwhmArr, xs, ys) f = open(outputTxtFileName+'-fwhm-grid.txt', 'w') f.write("# %s visit %s\n" % (" ".join(title), visit)) for xline, yline, fwhmline, ndataline \ in zip(xs.tolist(), ys.tolist(), fwhmResampled.tolist(), ndataGrids): for xx, yy, fwhm, ndata in zip(xline, yline, fwhmline, ndataline): if fwhm is None: fwhm = -9999 f.write('%f %f %f %d\n' % (xx, yy, fwhm, ndata)) elif showEllipticity: title.append("Ellipticity") scale = 4 if ellipticityDirection: # we don't care about the magnitude ellArr = 0.1 u = -ellArr*np.cos(paArr) v = -ellArr*np.sin(paArr) if gridPoints > 0: u = griddata(xArr, yArr, u, xs, ys) v = griddata(xArr, yArr, v, xs, ys) x, y = xs, ys else: x, y = xArr, yArr Q = plt.quiver(x, y, u, v, scale=scale, pivot="middle", headwidth=0, headlength=0, headaxislength=0, ) keyLen = 0.10 if not ellipticityDirection: # we care about the magnitude plt.quiverkey(Q, 0.20, 0.95, keyLen, "e=%g" % keyLen, labelpos='W') if outputTxtFileName: ndataGrids = getNumDataGrids(xArr, yArr, ellArr, xs, ys) f = open(outputTxtFileName+'-ell-grid.txt', 'w') f.write("# %s visit %s\n" % (" ".join(title), visit)) # f.write('# %f %f %f %f %f %f %f\n' % (x, y, ell, fwhm, pa, a, b)) for xline, yline, uline, vline, ndataline \ in zip(x.tolist(), y.tolist(), u.tolist(), v.tolist(), ndataGrids): for xx, yy, uu, vv, ndata in zip(xline, yline, uline, vline, ndataline): if uu is None: uu = -9999 if vv is None: vv = -9999 f.write('%f %f %f %f %d\n' % (xx, yy, uu, vv, ndata)) elif showNdataFwhm: title.append("N per fwhm grid") if len(xs) > 0: ndataGrids = getNumDataGrids(xArr, yArr, fwhmArr, xs, ys) plt.imshow(ndataGrids, interpolation='nearest', extent=extent, vmin=minNdata, vmax=maxNdata, origin='lower') plt.colorbar() else: pass elif showNdataEll: title.append("N per ell grid") if len(xs) > 0: ndataGrids = getNumDataGrids(xArr, yArr, ellArr, xs, ys) plt.imshow(ndataGrids, interpolation='nearest', extent=extent, vmin=minNdata, vmax=maxNdata, origin='lower') plt.colorbar() else: pass # plt.plot(xArr, yArr, "r.") # plt.plot(xs, ys, "b.") plt.axes().set_aspect('equal') plt.axis([-20000, 20000, -20000, 20000]) def frameInfoFrom(filepath): with fits.open(filepath) as hdul: h = hdul[0].header # 'object=ABELL2163 filter=HSC-I exptime=360.0 alt=62.11143274 ' # ' azm=202.32265181 hst=(23:40:08.363-23:40:48.546)' return 'object=%s filter=%s exptime=%.1f azm=%.2f hst=%s' % \ (h['OBJECT'], h['FILTER01'], h['EXPTIME'], h['AZIMUTH'], h['HST']) title.insert(0, frameInfoFrom(butler.get('raw_filename', {'visit': visit, 'ccd': 0})[0])) title.append(r'$\langle$FWHM$\rangle %4.2f$"' % np.median(fwhmArr)) plt.title("%s visit=%s" % (" ".join(title), visit), fontsize=9) return plt
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """Determine a PCA PSF model for an exposure given a list of PSF candidates. Parameters ---------- exposure : `lsst.afw.image.Exposure` Exposure containing the psf candidates. psfCandidateList : `list` of `lsst.meas.algorithms.PsfCandidate` A sequence of PSF candidates typically obtained by detecting sources and then running them through a star selector. metadata : `lsst.daf.base import PropertyList` or `None`, optional A home for interesting tidbits of information. flagKey : `str`, optional Schema key used to mark sources actually used in PSF determination. Returns ------- psf : `lsst.meas.algorithms.PcaPsf` The measured PSF. psfCellSet : `lsst.afw.math.SpatialCellSet` The PSF candidates. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info(__name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info(__name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) # match Kernel amplitudes for spatial plots matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes # Keep matplotlib alive post mortem keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info(__name__).showBadCandidates # Include bad candidates # Normalize residuals by object amplitude normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals pause = lsstDebug.Info(__name__).pause # Prompt user after each iteration? if display: afwDisplay.setDefaultMaskTransparency(75) if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwGeom.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.log.warn("WARNING: NOT scaling kernelSize by stellar quadrupole moment " "because config.kernelSize=%s >= 15; " "using config.kernelSize as as the width, instead", self.config.kernelSize) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2*int(self.config.kernelSize*math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print("Median size=%s" % (medSize,)) self.log.trace("Kernel size=%s", actualKernelSize) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print("Removing %d blended Psf candidates" % len(blendedCandidates)) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: if displayExposure: disp = afwDisplay.Display(frame=0) disp.mtv(exposure, title="psf determination") utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=afwDisplay.CYAN, ctypeUnused=afwDisplay.YELLOW, size=4, display=disp) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iterNum in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates): # maybe include bad candidates try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append((im, "%d%s" % (utils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception: continue if len(stamps) == 0: print("WARNING: No PSF candidates to show; try setting showBadCandidates=True") else: mos = afwDisplay.utils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append(im, label, (afwDisplay.GREEN if status == afwMath.SpatialCellCandidate.GOOD else afwDisplay.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else afwDisplay.RED)) disp8 = afwDisplay.Display(frame=8) mos.makeMosaic(display=disp8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.log.debug("chi^2=%s; id=%s", cand.getChi2(), cand.getSource().getId()) for cand in awfulCandidates: if display: print("Removing bad candidate: id=%d, chi^2=%f" % (cand.getSource().getId(), cand.getChi2())) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates rchi2 = cand.getChi2() # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = psf.getKernel() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception: continue fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p*k.getSum() predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters())] residuals.append([a/amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:, k].mean() rms = residuals[:, k].std() elif False: # Using interquartile range sr = numpy.sort(residuals[:, k]) mean = (sr[int(0.5*len(sr))] if len(sr)%2 else 0.5*(sr[int(0.5*len(sr))] + sr[int(0.5*len(sr)) + 1])) rms = 0.74*(sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))]) else: stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP) mean = stats.getValue(afwMath.MEANCLIP) rms = stats.getValue(afwMath.STDEVCLIP) rms = max(1.0e-4, rms) # Don't trust RMS below this due to numerical issues if display: print("Mean for component %d is %f" % (k, mean)) print("RMS for component %d is %f" % (k, rms)) badCandidates = list() for i, cand in enumerate(candidates): if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject*rms: badCandidates.append(i) badCandidates.sort(key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates): cand = candidates[c] if display: print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k, residuals[badCandidates[i], k], self.config.spatialReject*rms)) cand.setStatus(afwMath.SpatialCellCandidate.BAD) # # Display results # if display and displayIterations: if displayExposure: if iterNum > 0: disp.erase() utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", size=8, display=disp, ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, ctypeUnused=afwDisplay.MAGENTA) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", size=10, display=disp, ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED) if displayResiduals: while True: try: disp4 = afwDisplay.Display(frame=4) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) disp5 = afwDisplay.Display(frame=5) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except Exception: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) utils.showPsf(psf, eigenValues, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp7.scale('linear', 0, 1) if displayPsfSpatialModel: utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = input("Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:] else: reply = "" if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"): if reply == "c": pause = False elif reply == "h": print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " "s[ave fileName] y[es]") continue elif reply == "p": import pdb pdb.set_trace() elif reply == "q": display = False elif reply == "Q": sys.exit(1) elif reply == "s": fileName = args.pop(0) if not fileName: print("Please provide a filename") continue print("Saving to %s" % fileName) utils.saveSpatialCellSet(psfCellSet, fileName=fileName) continue break else: print("Unrecognised response: %s" % reply, file=sys.stderr) if reply == "n": break # One last time, to take advantage of the last iteration psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # Display code for debugging # if display and reply != "n": disp = afwDisplay.Display(frame=0) if displayExposure: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=8, display=disp) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: utils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", ctype=afwDisplay.YELLOW, ctypeBad=afwDisplay.RED, size=10, display=disp) if displayResiduals: disp4 = afwDisplay.Display(frame=4) utils.showPsfCandidates(exposure, psfCellSet, psf=psf, display=disp4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: disp6 = afwDisplay.Display(frame=6) utils.showPsf(psf, eigenValues, display=disp6) if displayPsfMosaic: disp7 = afwDisplay.Display(frame=7) utils.showPsfMosaic(exposure, psf, display=disp7, showFwhm=True) disp7.scale("linear", 0, 1) if displayPsfSpatialModel: utils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) # # Generate some QA information # # Count PSF stars # numGoodStars = 0 numAvailStars = 0 avgX = 0.0 avgY = 0.0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # don't ignore BAD stars numAvailStars += 1 for cand in cell.begin(True): # do ignore BAD stars src = cand.getSource() if flagKey is not None: src.set(flagKey, True) avgX += src.getX() avgY += src.getY() numGoodStars += 1 avgX /= numGoodStars avgY /= numGoodStars if metadata is not None: metadata.set("spatialFitChi2", fitChi2) metadata.set("numGoodStars", numGoodStars) metadata.set("numAvailStars", numAvailStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psf = PcaPsf(psf.getKernel(), lsst.geom.Point2D(avgX, avgY)) return psf, psfCellSet
def setUp(self): ixx, iyy, ixy = 1.0, 1.0, 0.0 self.data = afwGeom.Quadrupole(ixx, iyy, ixy)
def selectSources(self, sourceCat, matches=None, exposure=None): """Return a selection of PSF candidates that represent likely stars. A list of PSF candidates may be used by a PSF fitter to construct a PSF. Parameters: ----------- sourceCat : `lsst.afw.table.SourceCatalog` Catalog of sources to select from. This catalog must be contiguous in memory. matches : `list` of `lsst.afw.table.ReferenceMatch` or None Ignored in this SourceSelector. exposure : `lsst.afw.image.Exposure` or None The exposure the catalog was built from; used to get the detector to transform to TanPix, and for debug display. Return ------ struct : `lsst.pipe.base.Struct` The struct contains the following data: - selected : `array` of `bool`` Boolean array of sources that were selected, same length as sourceCat. """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells plotMagSize = lsstDebug.Info( __name__).plotMagSize # display the magnitude-size relation dumpData = lsstDebug.Info( __name__).dumpData # dump data to pickle file? detector = None pixToTanPix = None if exposure: detector = exposure.getDetector() if detector: pixToTanPix = detector.getTransform(PIXELS, TAN_PIXELS) # # Look at the distribution of stars in the magnitude-size plane # flux = sourceCat.get(self.config.sourceFluxField) fluxErr = sourceCat.get(self.config.sourceFluxField + "Err") xx = numpy.empty(len(sourceCat)) xy = numpy.empty_like(xx) yy = numpy.empty_like(xx) for i, source in enumerate(sourceCat): Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy() if pixToTanPix: p = lsst.geom.Point2D(source.getX(), source.getY()) linTransform = afwGeom.linearizeTransform(pixToTanPix, p).getLinear() m = afwGeom.Quadrupole(Ixx, Iyy, Ixy) m.transform(linTransform) Ixx, Iyy, Ixy = m.getIxx(), m.getIyy(), m.getIxy() xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy width = numpy.sqrt(0.5 * (xx + yy)) with numpy.errstate(invalid="ignore"): # suppress NAN warnings bad = reduce(lambda x, y: numpy.logical_or(x, sourceCat.get(y)), self.config.badFlags, False) bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width))) bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux))) if self.config.doFluxLimit: bad = numpy.logical_or(bad, flux < self.config.fluxMin) if self.config.fluxMax > 0: bad = numpy.logical_or(bad, flux > self.config.fluxMax) if self.config.doSignalToNoiseLimit: bad = numpy.logical_or( bad, flux / fluxErr < self.config.signalToNoiseMin) if self.config.signalToNoiseMax > 0: bad = numpy.logical_or( bad, flux / fluxErr > self.config.signalToNoiseMax) bad = numpy.logical_or(bad, width < self.config.widthMin) bad = numpy.logical_or(bad, width > self.config.widthMax) good = numpy.logical_not(bad) if not numpy.any(good): raise RuntimeError( "No objects passed our cuts for consideration as psf stars") mag = -2.5 * numpy.log10(flux[good]) width = width[good] # # Look for the maximum in the size histogram, then search upwards for the minimum that separates # the initial peak (of, we presume, stars) from the galaxies # if dumpData: import os import pickle as pickle _ii = 0 while True: pickleFile = os.path.expanduser( os.path.join("~", "widths-%d.pkl" % _ii)) if not os.path.exists(pickleFile): break _ii += 1 with open(pickleFile, "wb") as fd: pickle.dump(mag, fd, -1) pickle.dump(width, fd, -1) centers, clusterId = _kcenters( width, nCluster=4, useMedian=True, widthStdAllowed=self.config.widthStdAllowed) if display and plotMagSize: fig = plot( mag, width, centers, clusterId, magType=self.config.sourceFluxField.split(".")[-1].title(), marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True) else: fig = None clusterId = _improveCluster( width, centers, clusterId, nsigma=self.config.nSigmaClip, widthStdAllowed=self.config.widthStdAllowed) if display and plotMagSize: plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False) stellar = (clusterId == 0) # # We know enough to plot, if so requested # frame = 0 if fig: if display and displayExposure: disp = afwDisplay.Display(frame=frame) disp.mtv(exposure.getMaskedImage(), title="PSF candidates") global eventHandler eventHandler = EventHandler(fig.get_axes()[0], mag, width, sourceCat.getX()[good], sourceCat.getY()[good], frames=[frame]) fig.show() while True: try: reply = input("continue? [c h(elp) q(uit) p(db)] ").strip() except EOFError: reply = None if not reply: reply = "c" if reply: if reply[0] == "h": print("""\ We cluster the points; red are the stellar candidates and the other colours are other clusters. Points labelled + are rejects from the cluster (only for cluster 0). At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in the image display. """) elif reply[0] == "p": import pdb pdb.set_trace() elif reply[0] == 'q': sys.exit(1) else: break if display and displayExposure: mi = exposure.getMaskedImage() with disp.Buffering(): for i, source in enumerate(sourceCat): if good[i]: ctype = afwDisplay.GREEN # star candidate else: ctype = afwDisplay.RED # not star disp.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(), ctype=ctype) # stellar only applies to good==True objects mask = good == True # noqa (numpy bool comparison): E712 good[mask] = stellar return Struct(selected=good)