def showPsf(psf, eigenValues=None, XY=None, normalize=True, frame=None): """Display a PSF's eigen images If normalize is True, set the largest absolute value of each eigenimage to 1.0 (n.b. sum == 0.0 for i > 0) """ if eigenValues: coeffs = eigenValues elif XY is not None: coeffs = psf.getLocalKernel(afwGeom.PointD(XY[0], XY[1])).getKernelParameters() else: coeffs = None mos = displayUtils.Mosaic(gutter=2, background=-0.1) for i, k in enumerate(afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList()): im = afwImage.ImageD(k.getDimensions()) k.computeImage(im, False) if normalize: im /= numpy.max(numpy.abs(im.getArray())) if coeffs: mos.append(im, "%g" % (coeffs[i]/coeffs[0])) else: mos.append(im) mos.makeMosaic(frame=frame, title="Kernel Basis Functions") return mos
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128, matchKernelAmplitudes=False, keepPlots=True): """Plot the PSF spatial model.""" if plt is None: print >> sys.stderr, "Unable to import matplotlib" return noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) candPos = list() candFits = list() badPos = list() badFits = list() candAmps = list() badAmps = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) if not showBadCandidates and cand.isBad(): continue candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage() except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() targetFits = badFits if cand.isBad() else candFits targetPos = badPos if cand.isBad() else candPos targetAmps = badAmps if cand.isBad() else candAmps targetFits.append([x / amp for x in params]) targetPos.append(candCenter) targetAmps.append(amp)
print "Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum()
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = long(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: print templateIdList template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution ctr = afwGeom.Box2D(exposure.getBBox()).getCenter() psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr)) scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) # sigma of PSF of template image before warping ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter() psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr)) templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn("Src product does not exist; running detection, measurement, selection") # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma = scienceSigmaPost, doSmooth = not self.doPreConvolve, idFactory = idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len(makeKernelBasisList(self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds, False) else: raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars(exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [k for i,k in enumerate(kernelSources) if i % self.config.controlStepSize] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory ) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId() : x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [m.first.get(refCoordKey) for m in wcsResults.matches] residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches] colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) + 2.5*numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys()]) dlong = numpy.array([r[0].asArcseconds() for s,r in zip(sids, cresiduals) if s in srcToMatch.keys()]) dlat = numpy.array([r[1].asArcseconds() for s,r in zip(sids, cresiduals) if s in srcToMatch.keys()]) idx1 = numpy.where(colors<self.sourceSelector.config.grMin) idx2 = numpy.where((colors>=self.sourceSelector.config.grMin)& (colors<=self.sourceSelector.config.grMax)) idx3 = numpy.where(colors>self.sourceSelector.config.grMax) rms1Long = IqrToSigma*(numpy.percentile(dlong[idx1],75)-numpy.percentile(dlong[idx1],25)) rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1],75)-numpy.percentile(dlat[idx1],25)) rms2Long = IqrToSigma*(numpy.percentile(dlong[idx2],75)-numpy.percentile(dlong[idx2],25)) rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2],75)-numpy.percentile(dlat[idx2],25)) rms3Long = IqrToSigma*(numpy.percentile(dlong[idx3],75)-numpy.percentile(dlong[idx3],25)) rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3],75)-numpy.percentile(dlat[idx3],25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference #Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister ) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Running diaSource detection") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve ) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") self.measurement.run(diaSources, subtractedExposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn("Src product does not exist; cannot match with diaSources") srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = measAstrom.AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = measAstrom.AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn("No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \ refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if srcMatchDict.has_key(sid): diaSource.set("srcMatchId", srcMatchDict[sid]) if refMatchDict.has_key(sid): diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cast_KernelCandidateF(cand)) # Get basis list to build control sample kernels basisList = afwMath.cast_LinearCombinationKernel( kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True, fitBasisComponents=False, variance=None, chi=None): """Display the PSF candidates. If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs (and residuals) If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi If fitBasisComponents is true, also find the best linear combination of the PSF's components (if they exist) """ if chi is None: if variance is not None: # old name for chi chi = variance # # Show us the ccandidates # mos = displayUtils.Mosaic() # candidateCenters = [] candidateCentersBad = [] candidateIndex = 0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) rchi2 = cand.getChi2() if rchi2 > 1e100: rchi2 = numpy.nan if not showBadCandidates and cand.isBad(): continue if psf: im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x") try: im = cand.getMaskedImage() # copy of this object's image xc, yc = cand.getXCenter(), cand.getYCenter() margin = 0 if True else 5 w, h = im.getDimensions() bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions()) if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue()) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim *= stdev var = bim.getVariance(); var.set(stdev**2); del var sbim = im.Factory(bim, bbox) sbim <<= im del sbim im = bim xc += margin; yc += margin im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) except: continue if not variance: im_resid.append(im.Factory(im, True)) if True: # tweak up centroids mi = im psfIm = mi.getImage() config = measAlg.SourceMeasurementConfig() config.centroider.name = "centroid.sdss" config.slots.centroid = config.centroider.name schema = afwTable.SourceTable.makeMinimalSchema() measureSources = config.makeMeasureSources(schema) catalog = afwTable.SourceCatalog(schema) config.slots.setupTable(catalog.table) extra = 10 # enough margin to run the sdss centroider miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra) miBig[extra:-extra, extra:-extra] = mi miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra) mi = miBig; del miBig exp = afwImage.makeExposure(mi) exp.setPsf(psf) footprintSet = afwDet.FootprintSet(mi, afwDet.Threshold(0.5*numpy.max(psfIm.getArray())), "DETECTED") footprintSet.makeSources(catalog) if len(catalog) == 0: raise RuntimeError("Failed to detect any objects") elif len(catalog) == 1: source = catalog[0] else: # more than one source; find the once closest to (xc, yc) for i, s in enumerate(catalog): d = numpy.hypot(xc - s.getX(), yc - s.getY()) if i == 0 or d < dmin: source, dmin = s, d measureSources.applyWithPeak(source, exp) xc, yc = source.getCentroid() # residuals using spatial model try: chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc) except: chi2 = numpy.nan continue resid = im if variance: resid = resid.getImage() var = im.getVariance() var = var.Factory(var, True) numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt resid /= var im_resid.append(resid) # Fit the PSF components directly to the data (i.e. ignoring the spatial model) if fitBasisComponents: im = cand.getMaskedImage() im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = afwMath.KernelList(fit[1]) outputKernel = afwMath.LinearCombinationKernel(kernels, params) outImage = afwImage.ImageD(outputKernel.getDimensions()) outputKernel.computeImage(outImage, False) im -= outImage.convertF() resid = im if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim *= stdev sbim = im.Factory(bim, bbox) sbim <<= resid del sbim resid = bim if variance: resid = resid.getImage() resid /= var im_resid.append(resid) im = im_resid.makeMosaic() else: im = cand.getMaskedImage() if normalize: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() objId = splitId(cand.getSource().getId(), True)["objId"] if psf: lab = "%d chi^2 %.1f" % (objId, rchi2) ctype = ds9.RED if cand.isBad() else ds9.GREEN else: lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux()) ctype = ds9.GREEN mos.append(im, lab, ctype) if False and numpy.isnan(rchi2): ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1) print "amp", cand.getAmplitude() im = cand.getMaskedImage() center = (candidateIndex, xc - im.getX0(), yc - im.getY0()) candidateIndex += 1 if cand.isBad(): candidateCentersBad.append(center) else: candidateCenters.append(center) if variance: title = "chi(Psf fit)" else: title = "Stars & residuals" mosaicImage = mos.makeMosaic(frame=frame, title=title) with ds9.Buffering(): for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)): for cen in centers: bbox = mos.getBBox(cen[0]) ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color) return mosaicImage
def testGetPcaKernel(self): """Convert our cellSet to a LinearCombinationKernel""" nEigenComponents = 2 spatialOrder = 1 kernelSize = 21 nStarPerCell = 2 nStarPerCellSpatialFit = 2 tolerance = 1e-5 if display: ds9.mtv(self.mi, frame=0) # # Show the candidates we're using # for cell in self.cellSet.getCellList(): i = 0 for cand in cell: i += 1 source = algorithms.cast_PsfCandidateF(cand).getSource() xc, yc = source.getXAstrom() - self.mi.getX0(), source.getYAstrom() - self.mi.getY0() if i <= nStarPerCell: ds9.dot("o", xc, yc, ctype=ds9.GREEN) else: ds9.dot("o", xc, yc, ctype=ds9.YELLOW) pair = algorithms.createKernelFromPsfCandidates(self.cellSet, self.exposure.getDimensions(), self.exposure.getXY0(), nEigenComponents, spatialOrder, kernelSize, nStarPerCell) kernel, eigenValues = pair[0], pair[1]; del pair print "lambda", " ".join(["%g" % l for l in eigenValues]) pair = algorithms.fitSpatialKernelFromPsfCandidates(kernel, self.cellSet, nStarPerCellSpatialFit, tolerance) status, chi2 = pair[0], pair[1]; del pair print "Spatial fit: %s chi^2 = %.2g" % (status, chi2) psf = algorithms.PcaPsf.swigConvert(roundTripPsf(5, algorithms.PcaPsf(kernel))) # Hurrah! self.assertTrue(afwMath.cast_AnalyticKernel(psf.getKernel()) is None) self.assertTrue(afwMath.cast_LinearCombinationKernel(psf.getKernel()) is not None) self.checkTablePersistence(psf) if display: #print psf.getKernel().toString() eImages = [] for k in afwMath.cast_LinearCombinationKernel(psf.getKernel()).getKernelList(): im = afwImage.ImageD(k.getDimensions()) k.computeImage(im, False) eImages.append(im) mos = displayUtils.Mosaic() frame = 3 ds9.mtv(mos.makeMosaic(eImages), frame=frame) ds9.dot("Eigen Images", 0, 0, frame=frame) # # Make a mosaic of PSF candidates # stamps = [] stampInfo = [] for cell in self.cellSet.getCellList(): for cand in cell: # # Swig doesn't know that we inherited from SpatialCellMaskedImageCandidate; all # it knows is that we have a SpatialCellCandidate, and SpatialCellCandidates # don't know about getMaskedImage; so cast the pointer to PsfCandidate # cand = algorithms.cast_PsfCandidateF(cand) s = cand.getSource() im = cand.getMaskedImage() stamps.append(im) stampInfo.append("[%d 0x%x]" % (s.getId(), s.getFlagForDetection())) mos = displayUtils.Mosaic() frame = 1 ds9.mtv(mos.makeMosaic(stamps), frame=frame, lowOrderBits=True) for i in range(len(stampInfo)): ds9.dot(stampInfo[i], mos.getBBox(i).getX0(), mos.getBBox(i).getY0(), frame=frame, ctype=ds9.RED) psfImages = [] labels = [] if False: nx, ny = 3, 4 for iy in range(ny): for ix in range(nx): x = int((ix + 0.5)*self.mi.getWidth()/nx) y = int((iy + 0.5)*self.mi.getHeight()/ny) im = psf.getImage(x, y) psfImages.append(im.Factory(im, True)) labels.append("PSF(%d,%d)" % (int(x), int(y))) if True: print x, y, "PSF parameters:", psf.getKernel().getKernelParameters() else: nx, ny = 2, 2 for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]: im = psf.computeImage(afwGeom.PointD(x, y)) psfImages.append(im.Factory(im, True)) labels.append("PSF(%d,%d)" % (int(x), int(y))) if True: print x, y, "PSF parameters:", psf.getKernel().getKernelParameters() frame = 2 mos.makeMosaic(psfImages, frame=frame, mode=nx) mos.drawLabels(labels, frame=frame) if display: ds9.mtv(self.mi, frame=0) psfImages = [] labels = [] if False: nx, ny = 3, 4 for iy in range(ny): for ix in range(nx): x = int((ix + 0.5)*self.mi.getWidth()/nx) y = int((iy + 0.5)*self.mi.getHeight()/ny) algorithms.subtractPsf(psf, self.mi, x, y) else: nx, ny = 2, 2 for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]: if False: # Test subtraction with non-centered psfs x += 0.5; y -= 0.5 #algorithms.subtractPsf(psf, self.mi, x, y) ds9.mtv(self.mi, frame=1)
def plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=True, numSample=128, matchKernelAmplitudes=False, keepPlots=True): """Plot the PSF spatial model.""" if not plt: print >> sys.stderr, "Unable to import matplotlib" return noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) candPos = list() candFits = list() badPos = list() badFits = list() candAmps = list() badAmps = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) if not showBadCandidates and cand.isBad(): continue candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage() except Exception: continue fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() targetFits = badFits if cand.isBad() else candFits targetPos = badPos if cand.isBad() else candPos targetAmps = badAmps if cand.isBad() else candAmps targetFits.append([x / amp for x in params]) targetPos.append(candCenter) targetAmps.append(amp) numCandidates = len(candFits) numBasisFuncs = noSpatialKernel.getNBasisKernels() xGood = numpy.array([pos.getX() for pos in candPos]) - exposure.getX0() yGood = numpy.array([pos.getY() for pos in candPos]) - exposure.getY0() zGood = numpy.array(candFits) ampGood = numpy.array(candAmps) xBad = numpy.array([pos.getX() for pos in badPos]) - exposure.getX0() yBad = numpy.array([pos.getY() for pos in badPos]) - exposure.getY0() zBad = numpy.array(badFits) ampBad = numpy.array(badAmps) numBad = len(badPos) xRange = numpy.linspace(0, exposure.getWidth(), num=numSample) yRange = numpy.linspace(0, exposure.getHeight(), num=numSample) kernel = psf.getKernel() nKernelComponents = kernel.getNKernelParameters() # # Figure out how many panels we'll need # nPanelX = int(math.sqrt(nKernelComponents)) nPanelY = nKernelComponents//nPanelX while nPanelY*nPanelX < nKernelComponents: nPanelX += 1 fig = plt.figure(1) fig.clf() try: fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word except: # protect against API changes pass # # Generator for axes arranged in panels # subplots = makeSubplots(fig, 2, 2, Nx=nPanelX, Ny=nPanelY, xgutter=0.06, ygutter=0.06, pygutter=0.04) for k in range(nKernelComponents): func = kernel.getSpatialFunction(k) dfGood = zGood[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in candPos]) yMin = dfGood.min() yMax = dfGood.max() if numBad > 0: dfBad = zBad[:,k] - numpy.array([func(pos.getX(), pos.getY()) for pos in badPos]) yMin = min([yMin, dfBad.min()]) yMax = max([yMax, dfBad.max()]) yMin -= 0.05 * (yMax - yMin) yMax += 0.05 * (yMax - yMin) yMin = -0.01 yMax = 0.01 fRange = numpy.ndarray((len(xRange), len(yRange))) for j, yVal in enumerate(yRange): for i, xVal in enumerate(xRange): fRange[j][i] = func(xVal, yVal) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() ax.set_autoscale_on(False) ax.set_xbound(lower=0, upper=exposure.getHeight()) ax.set_ybound(lower=yMin, upper=yMax) ax.plot(yGood, dfGood, 'b+') if numBad > 0: ax.plot(yBad, dfBad, 'r+') ax.axhline(0.0) ax.set_title('Residuals(y)') #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() if matchKernelAmplitudes and k == 0: vmin = 0.0 vmax = 1.1 else: vmin = fRange.min() vmax = fRange.max() norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax) im = ax.imshow(fRange, aspect='auto', origin="lower", norm=norm, extent=[0, exposure.getWidth()-1, 0, exposure.getHeight()-1]) ax.set_title('Spatial poly') plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax]) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() ax.set_autoscale_on(False) ax.set_xbound(lower=0, upper=exposure.getWidth()) ax.set_ybound(lower=yMin, upper=yMax) ax.plot(xGood, dfGood, 'b+') if numBad > 0: ax.plot(xBad, dfBad, 'r+') ax.axhline(0.0) ax.set_title('K%d Residuals(x)' % k) #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- ax = subplots.next() if False: ax.scatter(xGood, yGood, c=dfGood, marker='o') ax.scatter(xBad, yBad, c=dfBad, marker='x') ax.set_xbound(lower=0, upper=exposure.getWidth()) ax.set_ybound(lower=0, upper=exposure.getHeight()) ax.set_title('Spatial residuals') plt.colorbar(im, orientation='horizontal') else: calib = exposure.getCalib() if calib.getFluxMag0()[0] <= 0: calib = type(calib)() calib.setFluxMag0(1.0) with CalibNoThrow(): ax.plot(calib.getMagnitude(candAmps), zGood[:,k], 'b+') if numBad > 0: ax.plot(calib.getMagnitude(badAmps), zBad[:,k], 'r+') ax.set_title('Flux variation') fig.show() global keptPlots if keepPlots and not keptPlots: # Keep plots open when done def show(): print "%s: Please close plots when done." % __name__ try: plt.show() except: pass print "Plots closed, exiting..." import atexit atexit.register(show) keptPlots = True
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True, fitBasisComponents=False, variance=None, chi=None): """Display the PSF candidates. If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs (and residuals) If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi If fitBasisComponents is true, also find the best linear combination of the PSF's components (if they exist) """ if chi is None: if variance is not None: # old name for chi chi = variance # # Show us the ccandidates # mos = displayUtils.Mosaic() # candidateCenters = [] candidateCentersBad = [] candidateIndex = 0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) rchi2 = cand.getChi2() if rchi2 > 1e100: rchi2 = numpy.nan if not showBadCandidates and cand.isBad(): continue if psf: im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x") try: im = cand.getMaskedImage() # copy of this object's image xc, yc = cand.getXCenter(), cand.getYCenter() margin = 0 if True else 5 w, h = im.getDimensions() bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions()) if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue()) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim.getVariance().set(stdev**2) bim.assign(im, bbox) im = bim xc += margin yc += margin im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) except: continue if not variance: im_resid.append(im.Factory(im, True)) if True: # tweak up centroids mi = im psfIm = mi.getImage() config = measBase.SingleFrameMeasurementTask.ConfigClass() config.slots.centroid = "base_SdssCentroid" schema = afwTable.SourceTable.makeMinimalSchema() measureSources = measBase.SingleFrameMeasurementTask(schema, config=config) catalog = afwTable.SourceCatalog(schema) extra = 10 # enough margin to run the sdss centroider miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra) miBig[extra:-extra, extra:-extra] = mi miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra) mi = miBig del miBig exp = afwImage.makeExposure(mi) exp.setPsf(psf) footprintSet = afwDet.FootprintSet(mi, afwDet.Threshold(0.5*numpy.max(psfIm.getArray())), "DETECTED") footprintSet.makeSources(catalog) if len(catalog) == 0: raise RuntimeError("Failed to detect any objects") measureSources.run(catalog, exp) if len(catalog) == 1: source = catalog[0] else: # more than one source; find the once closest to (xc, yc) dmin = None # an invalid value to catch logic errors for i, s in enumerate(catalog): d = numpy.hypot(xc - s.getX(), yc - s.getY()) if i == 0 or d < dmin: source, dmin = s, d xc, yc = source.getCentroid() # residuals using spatial model try: chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc) except: chi2 = numpy.nan continue resid = im if variance: resid = resid.getImage() var = im.getVariance() var = var.Factory(var, True) numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt resid /= var im_resid.append(resid) # Fit the PSF components directly to the data (i.e. ignoring the spatial model) if fitBasisComponents: im = cand.getMaskedImage() im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) try: noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) except: noSpatialKernel = None if noSpatialKernel: candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = afwMath.KernelList(fit[1]) outputKernel = afwMath.LinearCombinationKernel(kernels, params) outImage = afwImage.ImageD(outputKernel.getDimensions()) outputKernel.computeImage(outImage, False) im -= outImage.convertF() resid = im if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim *= stdev bim.assign(resid, bbox) resid = bim if variance: resid = resid.getImage() resid /= var im_resid.append(resid) im = im_resid.makeMosaic() else: im = cand.getMaskedImage() if normalize: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() objId = splitId(cand.getSource().getId(), True)["objId"] if psf: lab = "%d chi^2 %.1f" % (objId, rchi2) ctype = ds9.RED if cand.isBad() else ds9.GREEN else: lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux()) ctype = ds9.GREEN mos.append(im, lab, ctype) if False and numpy.isnan(rchi2): ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1) print "amp", cand.getAmplitude() im = cand.getMaskedImage() center = (candidateIndex, xc - im.getX0(), yc - im.getY0()) candidateIndex += 1 if cand.isBad(): candidateCentersBad.append(center) else: candidateCenters.append(center) if variance: title = "chi(Psf fit)" else: title = "Stars & residuals" mosaicImage = mos.makeMosaic(frame=frame, title=title) with ds9.Buffering(): for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)): for cen in centers: bbox = mos.getBBox(cen[0]) ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color) return mosaicImage
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = long(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution ctr = afwGeom.Box2D(exposure.getBBox()).getCenter() psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr)) scienceSigmaOrig = psfAttr.computeGaussianWidth( psfAttr.ADAPTIVE_MOMENT) # sigma of PSF of template image before warping ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter() psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr)) templateSigma = psfAttr.computeGaussianWidth( psfAttr.ADAPTIVE_MOMENT) # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel( ).getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn( "Src product does not exist; running detection, measurement, selection" ) # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma=scienceSigmaPost, doSmooth=not self.doPreConvolve, idFactory=idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len( makeKernelBasisList( self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch( exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources matches = afwTable.matchRaDec(templateSources, selectSources, 1.0 * afwGeom.arcseconds, False) else: raise RuntimeError( "doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars( exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [ k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize ] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info( "Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId(): x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable( ).getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable( ).getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [ m.first.get(refCoordKey) for m in wcsResults.matches ] residuals = [ m.first.get(refCoordKey).getOffsetFrom( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [ m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] colors = numpy.array([ -2.5 * numpy.log10(srcToMatch[x].get("g")) + 2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys() ]) dlong = numpy.array([ r[0].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) dlat = numpy.array([ r[1].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) idx1 = numpy.where( colors < self.sourceSelector.config.grMin) idx2 = numpy.where( (colors >= self.sourceSelector.config.grMin) & (colors <= self.sourceSelector.config.grMax)) idx3 = numpy.where( colors > self.sourceSelector.config.grMax) rms1Long = IqrToSigma * (numpy.percentile( dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)) rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) - numpy.percentile(dlat[idx1], 25)) rms2Long = IqrToSigma * (numpy.percentile( dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)) rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) - numpy.percentile(dlat[idx2], 25)) rms3Long = IqrToSigma * (numpy.percentile( dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)) rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) - numpy.percentile(dlat[idx3], 25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info( "Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference #Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Computing diffim PSF") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run( exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False. if self.config.doDecorrelation and self.config.doSubtract: decorrResult = self.decorrelate.run(exposure, templateExposure, subtractedExposure, subtractRes.psfMatchingKernel) subtractedExposure = decorrResult.correctedExposure if self.config.doDetection: self.log.info("Running diaSource detection") # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if not self.config.doDipoleFitting: self.measurement.run(diaSources, subtractedExposure) else: if self.config.doSubtract: self.measurement.run(diaSources, subtractedExposure, exposure, subtractRes.matchedExposure) else: self.measurement.run(diaSources, subtractedExposure, exposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs( ).pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn( "Src product does not exist; cannot match with diaSources" ) srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = measAstrom.AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = measAstrom.AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn( "No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info( "Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \ refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if srcMatchDict.has_key(sid): diaSource.set("srcMatchId", srcMatchDict[sid]) if refMatchDict.has_key(sid): diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cast_KernelCandidateF(cand)) # Get basis list to build control sample kernels basisList = afwMath.cast_LinearCombinationKernel( kernelCandList[0].getKernel( KernelCandidateF.ORIG)).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )
class PcaPsfDeterminer(object): """! A measurePsfTask psf estimator """ ConfigClass = PcaPsfDeterminerConfig def __init__(self, config): """!Construct a PCA PSF Fitter \param[in] config instance of PcaPsfDeterminerConfig """ self.config = config # N.b. name of component is meas.algorithms.psfDeterminer so you can turn on psf debugging # independent of which determiner is active self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer") self.warnLog = pexLog.Log(pexLog.getDefaultLog(), "meas.algorithms.psfDeterminer") def _fitPsf(self, exposure, psfCellSet, kernelSize, nEigenComponents): algorithmsLib.PsfCandidateF.setPixelThreshold( self.config.pixelThreshold) algorithmsLib.PsfCandidateF.setMaskBlends(self.config.doMaskBlends) # # Loop trying to use nEigenComponents, but allowing smaller numbers if necessary # for nEigen in range(nEigenComponents, 0, -1): # Determine KL components try: kernel, eigenValues = algorithmsLib.createKernelFromPsfCandidates( psfCellSet, exposure.getDimensions(), exposure.getXY0(), nEigen, self.config.spatialOrder, kernelSize, self.config.nStarPerCell, bool(self.config.constantWeight)) break # OK, we can get nEigen components except pexExceptions.LengthError as e: if nEigen == 1: # can't go any lower raise IndexError("No viable PSF candidates survive") self.warnLog.log( pexLog.Log.WARN, "%s: reducing number of eigen components" % e.what()) # # We got our eigen decomposition so let's use it # # Express eigenValues in units of reduced chi^2 per star size = kernelSize + 2 * self.config.borderWidth nu = size * size - 1 # number of degrees of freedom/star for chi^2 eigenValues = [ l / float( algorithmsLib.countPsfCandidates( psfCellSet, self.config.nStarPerCell) * nu) for l in eigenValues ] # Fit spatial model status, chi2 = algorithmsLib.fitSpatialKernelFromPsfCandidates( kernel, psfCellSet, bool(self.config.nonLinearSpatialFit), self.config.nStarPerCellSpatialFit, self.config.tolerance, self.config.lam) psf = algorithmsLib.PcaPsf(kernel) return psf, eigenValues, nEigen, chi2 def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """!Determine a PCA PSF model for an exposure given a list of PSF candidates \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure) \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector \param[in,out] metadata a home for interesting tidbits of information \param[in] flagKey schema key used to mark sources actually used in PSF determination \return a list of - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info( __name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info( __name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info( __name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info( __name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info( __name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) matchKernelAmplitudes = lsstDebug.Info( __name__).matchKernelAmplitudes # match Kernel amplitudes # for spatial plots keepMatplotlibPlots = lsstDebug.Info( __name__).keepMatplotlibPlots # Keep matplotlib alive # post mortem displayPsfSpatialModel = lsstDebug.Info( __name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info( __name__).showBadCandidates # Include bad candidates normalizeResiduals = lsstDebug.Info( __name__).normalizeResiduals # Normalise residuals by # object amplitude pause = lsstDebug.Info( __name__).pause # Prompt user after each iteration? if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception, e: self.debugLog.debug( 2, "Skipping PSF candidate %d of %d: %s" % (i, len(psfCandidateList), e)) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.debugLog.debug(1, \ "WARNING: NOT scaling kernelSize by stellar quadrupole moment " + "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead" \ % (self.config.kernelSize,) ) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2 * int(self.config.kernelSize * math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print "Median size=%s" % (medSize, ) self.debugLog.debug(3, "Kernel size=%s" % (actualKernelSize, )) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [ ] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print "Removing %d blended Psf candidates" % len( blendedCandidates) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW, size=4, frame=frame) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iter in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates # import lsst.afw.display.utils as displayUtils stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates ): # maybe include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append( (im, "%d%s" % (maUtils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception, e: continue if len(stamps) == 0: print "WARNING: No PSF candidates to show; try setting showBadCandidates=True" else: mos = displayUtils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics( im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append( im, label, ds9.GREEN if status == afwMath.SpatialCellCandidate.GOOD else ds9.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED) mos.makeMosaic(frame=8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN ) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.debugLog.debug( 2, "chi^2=%s; id=%s" % (cand.getChi2(), cand.getSource().getId())) for cand in awfulCandidates: if display: print "Removing bad candidate: id=%d, chi^2=%f" % \ (cand.getSource().getId(), cand.getChi2()) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) rchi2 = cand.getChi2( ) # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = int( len(badCandidates) * (iter + 1) / self.config.nIterForPsf + 0.5) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print "Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = afwMath.cast_LinearCombinationKernel( psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage( noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() predict = [ kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters()) ] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append( [a / amp - p for a, p in zip(params, predict)]) candidates.append(cand)
def run(self, sensorRef): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = long(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() if self.config.useWinter2013Hacks and self.config.winter2013borderMask > 0: self.log.warn("USING WINTER2013 HACK: MASKING BORDER PIXELS") bbox = exposure.getBBox(afwImage.PARENT) bbox.grow(-self.config.winter2013borderMask) self.setEdgeBits(exposure.getMaskedImage(), bbox, exposure.getMaskedImage().getMask().getPlaneBitMask("NO_DATA")) # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution ctr = afwGeom.Box2D(exposure.getBBox(afwImage.PARENT)).getCenter() psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr)) scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: templateExposure, templateSources = self.getTemplate(exposure, sensorRef) # sigma of PSF of template image before warping ctr = afwGeom.Box2D(templateExposure.getBBox(afwImage.PARENT)).getCenter() psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr)) templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getKernel().getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = psf afwMath.convolve(destMI, srcMI, preConvPsf.getKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn("Src product does not exist; running detection, measurement, selection") # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma = scienceSigmaPost, doSmooth = not self.doPreConvolve, idFactory = idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len(makeKernelBasisList(self.subtract.config.kernel.active, referenceFwhmPix = scienceSigmaPost * FwhmPerSigma, targetFwhmPix = templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources self.kcQa = diUtils.KernelCandidateQa(nparam, self.log) selectSources = self.kcQa.addToSchema(selectSources) astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig()) astromRet = astrometer.useKnownWcs(selectSources, exposure=exposure) matches = astromRet.matches kernelSources = self.sourceSelector.selectSources(exposure, selectSources, matches=matches) random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] [kernelSources.remove(x) for x in controlSources] self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" \ % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # First step: we need to subtract the background out # for detection and measurement. Use large binsize # for the background estimation. binsize = self.config.templateBgBinSize # Second step: we need to run detection on the # background-subtracted template # # Estimate FWHM for detection templateSources = self.subtract.getSelectSources( templateExposure, sigma = templateSigma, doSmooth = True, idFactory = idFactory, binsize = binsize, ) # Third step: we need to fit the relative astrometry. # # One problem is that the SIP fits are w.r.t. CRPIX, # and these coadd patches have the CRPIX of the entire # tract, i.e. off the image. This causes # register.fitWcs to fail. A workaround for now is to # re-fit the Wcs which returns with a CRPIX that is on # the image, and *then* to fit for the relative Wcs. # # Requires low Sip order to avoid overfitting # useWinter2013Hacks includes us using the deep calexp # as the template. In this case we don't need to # refit the Wcs. if not self.config.useWinter2013Hacks: sipOrder = self.config.templateSipOrder astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(sipOrder=sipOrder)) newWcs = astrometer.determineWcs(templateSources, templateExposure).getWcs() results = self.register.run(templateSources, newWcs, templateExposure.getBBox(afwImage.PARENT), selectSources) else: if self.config.winter2013WcsShift > 0.0: offset = afwGeom.Extent2D(self.config.winter2013WcsShift, self.config.winter2013WcsShift) cKey = templateSources[0].getTable().getCentroidKey() for source in templateSources: centroid = source.get(cKey) source.set(cKey, centroid+offset) elif self.config.winter2013WcsRms > 0.0: cKey = templateSources[0].getTable().getCentroidKey() for source in templateSources: offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(), self.config.winter2013WcsRms*numpy.random.normal()) centroid = source.get(cKey) source.set(cKey, centroid+offset) results = self.register.run(templateSources, templateExposure.getWcs(), templateExposure.getBBox(afwImage.PARENT), selectSources) warpedExp = self.register.warpExposure(templateExposure, results.wcs, exposure.getWcs(), exposure.getBBox(afwImage.PARENT)) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: refCoordKey = results.matches[0].first.getTable().getCoordKey() inCentroidKey = results.matches[0].second.getTable().getCentroidKey() sids = [m.first.getId() for m in results.matches] positions = [m.first.get(refCoordKey) for m in results.matches] residuals = [m.first.get(refCoordKey).getOffsetFrom( results.wcs.pixelToSky(m.second.get(inCentroidKey))) for m in results.matches] allresids = dict(zip(sids, zip(positions, residuals))) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference #Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure = templateExposure, scienceExposure = exposure, scienceFwhmPix = scienceSigmaPost * FwhmPerSigma, templateFwhmPix = templateSigma * FwhmPerSigma, candidateList = kernelSources, convolveTemplate = self.config.convolveTemplate, doWarping = not self.config.doUseRegister ) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Running diaSource detection") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: templateExposure, templateSources = self.getTemplate(exposure, sensorRef) subtractedExposure.setPsf(templateExposure.getPsf()) # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table = table, exposure = subtractedExposure, doSmooth = not self.config.doPreConvolve ) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if len(diaSources) < self.config.maxDiaSourcesToMeasure: self.dipolemeasurement.run(subtractedExposure, diaSources) else: self.measurement.run(subtractedExposure, diaSources) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds() # This does not do what I expect so I cobbled together a brute force method in python srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for \ srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn("Src product does not exist; cannot match with diaSources") srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(catalogMatchDist=matchRadAsec)) astromRet = astrometer.useKnownWcs(diaSources, exposure=exposure) refMatches = astromRet.matches if refMatches is None: self.log.warn("No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \ refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if srcMatchDict.has_key(sid): diaSource.set("srcMatchId", srcMatchDict[sid]) if refMatchDict.has_key(sid): diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sourceWriteFlags = (0 if self.config.doWriteHeavyFootprintsInSources else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS) sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc", flags=sourceWriteFlags) if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cast_KernelCandidateF(cand)) # Get basis list to build control sample kernels basisList = afwMath.cast_LinearCombinationKernel( kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList() controlCandList = \ diffimTools.sourceTableToCandList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, dobuild=True, basisList=basisList) self.kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) self.kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: self.kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: self.kcQa.aggregate(selectSources, self.metadata, allresids) #Persist using butler sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure = subtractedExposure, subtractRes = subtractRes, sources = diaSources, )
def showPsfCandidates(exposure, psfCellSet, psf=None, frame=None, normalize=True, showBadCandidates=True, variance=None, chi=None): """Display the PSF candidates. If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs (and residuals) If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi """ if chi is None: if variance is not None: # old name for chi chi = variance # # Show us the ccandidates # mos = displayUtils.Mosaic() # candidateCenters = [] candidateCentersBad = [] candidateIndex = 0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.cast_PsfCandidateF(cand) rchi2 = cand.getChi2() if rchi2 > 1e100: rchi2 = numpy.nan if not showBadCandidates and cand.isBad(): continue if psf: im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x") try: im = cand.getMaskedImage() # copy of this object's image xc, yc = cand.getXCenter(), cand.getYCenter() margin = 0 if True else 5 w, h = im.getDimensions() bbox = afwGeom.BoxI(afwGeom.PointI(margin, margin), im.getDimensions()) if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue()) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim *= stdev var = bim.getVariance(); var.set(stdev**2); del var sbim = im.Factory(bim, bbox) sbim <<= im del sbim im = bim xc += margin; yc += margin im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) except: continue if not variance: im_resid.append(im.Factory(im, True)) # residuals using spatial model chi2 = algorithmsLib.subtractPsf(psf, im, xc, yc) resid = im if variance: resid = resid.getImage() var = im.getVariance() var = var.Factory(var, True) numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt resid /= var im_resid.append(resid) # Fit the PSF components directly to the data (i.e. ignoring the spatial model) im = cand.getMaskedImage() im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) noSpatialKernel = afwMath.cast_LinearCombinationKernel(psf.getKernel()) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) fit = algorithmsLib.fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = afwMath.KernelList(fit[1]) outputKernel = afwMath.LinearCombinationKernel(kernels, params) outImage = afwImage.ImageD(outputKernel.getDimensions()) outputKernel.computeImage(outImage, False) im -= outImage.convertF() resid = im if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim *= stdev sbim = im.Factory(bim, bbox) sbim <<= resid del sbim resid = bim if variance: resid = resid.getImage() resid /= var im_resid.append(resid) im = im_resid.makeMosaic() else: im = cand.getMaskedImage() if normalize: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() objId = splitId(cand.getSource().getId(), True)["objId"] if psf: lab = "%d chi^2 %.1f" % (objId, rchi2) ctype = ds9.RED if cand.isBad() else ds9.GREEN else: lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfFlux()) ctype = ds9.GREEN mos.append(im, lab, ctype) if False and numpy.isnan(rchi2): ds9.mtv(cand.getMaskedImage().getImage(), title="candidate", frame=1) print "amp", cand.getAmplitude() im = cand.getMaskedImage() center = (candidateIndex, xc - im.getX0(), yc - im.getY0()) candidateIndex += 1 if cand.isBad(): candidateCentersBad.append(center) else: candidateCenters.append(center) if variance: title = "chi(Psf fit)" else: title = "Stars & residuals" mosaicImage = mos.makeMosaic(frame=frame, title=title) with ds9.Buffering(): for centers, color in ((candidateCenters, ds9.GREEN), (candidateCentersBad, ds9.RED)): for cen in centers: bbox = mos.getBBox(cen[0]) ds9.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), frame=frame, ctype=color) return mosaicImage
chi2) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = afwMath.cast_LinearCombinationKernel( psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.cast_PsfCandidateF(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception, e: continue fit = algorithmsLib.fitKernelParamsToImage( noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1]
def testGetPcaKernel(self): """Convert our cellSet to a LinearCombinationKernel""" nEigenComponents = 2 spatialOrder = 1 kernelSize = 21 nStarPerCell = 2 nStarPerCellSpatialFit = 2 tolerance = 1e-5 if display: ds9.mtv(self.mi, frame=0) # # Show the candidates we're using # for cell in self.cellSet.getCellList(): i = 0 for cand in cell: i += 1 source = algorithms.PsfCandidateF.cast(cand).getSource() xc, yc = source.getXAstrom() - self.mi.getX0( ), source.getYAstrom() - self.mi.getY0() if i <= nStarPerCell: ds9.dot("o", xc, yc, ctype=ds9.GREEN) else: ds9.dot("o", xc, yc, ctype=ds9.YELLOW) pair = algorithms.createKernelFromPsfCandidates( self.cellSet, self.exposure.getDimensions(), self.exposure.getXY0(), nEigenComponents, spatialOrder, kernelSize, nStarPerCell) kernel, eigenValues = pair[0], pair[1] del pair print("lambda", " ".join(["%g" % l for l in eigenValues])) pair = algorithms.fitSpatialKernelFromPsfCandidates( kernel, self.cellSet, nStarPerCellSpatialFit, tolerance) status, chi2 = pair[0], pair[1] del pair print("Spatial fit: %s chi^2 = %.2g" % (status, chi2)) psf = algorithms.PcaPsf.swigConvert( roundTripPsf(5, algorithms.PcaPsf(kernel))) # Hurrah! self.assertIsNone(afwMath.cast_AnalyticKernel(psf.getKernel())) self.assertIsNotNone( afwMath.cast_LinearCombinationKernel(psf.getKernel())) self.checkTablePersistence(psf) if display: # print psf.getKernel().toString() eImages = [] for k in afwMath.cast_LinearCombinationKernel( psf.getKernel()).getKernelList(): im = afwImage.ImageD(k.getDimensions()) k.computeImage(im, False) eImages.append(im) mos = displayUtils.Mosaic() frame = 3 ds9.mtv(mos.makeMosaic(eImages), frame=frame) ds9.dot("Eigen Images", 0, 0, frame=frame) # # Make a mosaic of PSF candidates # stamps = [] stampInfo = [] for cell in self.cellSet.getCellList(): for cand in cell: # # Swig doesn't know that we inherited from SpatialCellMaskedImageCandidate; all # it knows is that we have a SpatialCellCandidate, and SpatialCellCandidates # don't know about getMaskedImage; so cast the pointer to PsfCandidate # cand = algorithms.PsfCandidateF.cast(cand) s = cand.getSource() im = cand.getMaskedImage() stamps.append(im) stampInfo.append("[%d 0x%x]" % (s.getId(), s.getFlagForDetection())) mos = displayUtils.Mosaic() frame = 1 ds9.mtv(mos.makeMosaic(stamps), frame=frame, lowOrderBits=True) for i in range(len(stampInfo)): ds9.dot(stampInfo[i], mos.getBBox(i).getX0(), mos.getBBox(i).getY0(), frame=frame, ctype=ds9.RED) psfImages = [] labels = [] if False: nx, ny = 3, 4 for iy in range(ny): for ix in range(nx): x = int((ix + 0.5) * self.mi.getWidth() / nx) y = int((iy + 0.5) * self.mi.getHeight() / ny) im = psf.getImage(x, y) psfImages.append(im.Factory(im, True)) labels.append("PSF(%d,%d)" % (int(x), int(y))) if True: print((x, y, "PSF parameters:", psf.getKernel().getKernelParameters())) else: nx, ny = 2, 2 for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]: im = psf.computeImage(afwGeom.PointD(x, y)) psfImages.append(im.Factory(im, True)) labels.append("PSF(%d,%d)" % (int(x), int(y))) if True: print(x, y, "PSF parameters:", psf.getKernel().getKernelParameters()) frame = 2 mos.makeMosaic(psfImages, frame=frame, mode=nx) mos.drawLabels(labels, frame=frame) if display: ds9.mtv(self.mi, frame=0) psfImages = [] labels = [] if False: nx, ny = 3, 4 for iy in range(ny): for ix in range(nx): x = int((ix + 0.5) * self.mi.getWidth() / nx) y = int((iy + 0.5) * self.mi.getHeight() / ny) algorithms.subtractPsf(psf, self.mi, x, y) else: nx, ny = 2, 2 for x, y in [(20, 20), (60, 20), (60, 210), (20, 210)]: if False: # Test subtraction with non-centered psfs x += 0.5 y -= 0.5 #algorithms.subtractPsf(psf, self.mi, x, y) ds9.mtv(self.mi, frame=1)
def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None): """!Determine a PCA PSF model for an exposure given a list of PSF candidates \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure) \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate); typically obtained by detecting sources and then running them through a star selector \param[in,out] metadata a home for interesting tidbits of information \param[in] flagKey schema key used to mark sources actually used in PSF determination \return a list of - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates """ import lsstDebug display = lsstDebug.Info(__name__).display displayExposure = lsstDebug.Info( __name__).displayExposure # display the Exposure + spatialCells displayPsfCandidates = lsstDebug.Info( __name__).displayPsfCandidates # show the viable candidates displayIterations = lsstDebug.Info( __name__).displayIterations # display on each PSF iteration displayPsfComponents = lsstDebug.Info( __name__).displayPsfComponents # show the PCA components displayResiduals = lsstDebug.Info( __name__).displayResiduals # show residuals displayPsfMosaic = lsstDebug.Info( __name__).displayPsfMosaic # show mosaic of reconstructed PSF(x,y) # match Kernel amplitudes for spatial plots matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes # Keep matplotlib alive post mortem keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots displayPsfSpatialModel = lsstDebug.Info( __name__).displayPsfSpatialModel # Plot spatial model? showBadCandidates = lsstDebug.Info( __name__).showBadCandidates # Include bad candidates # Normalize residuals by object amplitude normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals pause = lsstDebug.Info( __name__).pause # Prompt user after each iteration? if display > 1: pause = True mi = exposure.getMaskedImage() if len(psfCandidateList) == 0: raise RuntimeError("No PSF candidates supplied.") # construct and populate a spatial cell set bbox = mi.getBBox() psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY) sizes = [] for i, psfCandidate in enumerate(psfCandidateList): if psfCandidate.getSource().getPsfFluxFlag(): # bad measurement continue try: psfCellSet.insertCandidate(psfCandidate) except Exception as e: self.log.debug("Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e) continue source = psfCandidate.getSource() quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy()) axes = afwEll.Axes(quad) sizes.append(axes.getA()) if len(sizes) == 0: raise RuntimeError("No usable PSF candidates supplied") nEigenComponents = self.config.nEigenComponents # initial version if self.config.kernelSize >= 15: self.log.warn( "WARNING: NOT scaling kernelSize by stellar quadrupole moment " + "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead", self.config.kernelSize) actualKernelSize = int(self.config.kernelSize) else: medSize = numpy.median(sizes) actualKernelSize = 2 * int(self.config.kernelSize * math.sqrt(medSize) + 0.5) + 1 if actualKernelSize < self.config.kernelSizeMin: actualKernelSize = self.config.kernelSizeMin if actualKernelSize > self.config.kernelSizeMax: actualKernelSize = self.config.kernelSizeMax if display: print("Median size=%s" % (medSize, )) self.log.trace("Kernel size=%s", actualKernelSize) # Set size of image returned around candidate psfCandidateList[0].setHeight(actualKernelSize) psfCandidateList[0].setWidth(actualKernelSize) if self.config.doRejectBlends: # Remove blended candidates completely blendedCandidates = [ ] # Candidates to remove; can't do it while iterating for cell, cand in candidatesIter(psfCellSet, False): if len(cand.getSource().getFootprint().getPeaks()) > 1: blendedCandidates.append((cell, cand)) continue if display: print("Removing %d blended Psf candidates" % len(blendedCandidates)) for cell, cand in blendedCandidates: cell.removeCandidate(cand) if sum(1 for cand in candidatesIter(psfCellSet, False)) == 0: raise RuntimeError("All PSF candidates removed as blends") if display: frame = 0 if displayExposure: ds9.mtv(exposure, frame=frame, title="psf determination") maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, symb="o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW, size=4, frame=frame) # # Do a PCA decomposition of those PSF candidates # reply = "y" # used in interactive mode for iterNum in range(self.config.nIterForPsf): if display and displayPsfCandidates: # Show a mosaic of usable PSF candidates # import lsst.afw.display.utils as displayUtils stamps = [] for cell in psfCellSet.getCellList(): for cand in cell.begin(not showBadCandidates ): # maybe include bad candidates cand = algorithmsLib.PsfCandidateF.cast(cand) try: im = cand.getMaskedImage() chi2 = cand.getChi2() if chi2 > 1e100: chi2 = numpy.nan stamps.append( (im, "%d%s" % (maUtils.splitId(cand.getSource().getId(), True)["objId"], chi2), cand.getStatus())) except Exception as e: continue if len(stamps) == 0: print( "WARNING: No PSF candidates to show; try setting showBadCandidates=True" ) else: mos = displayUtils.Mosaic() for im, label, status in stamps: im = type(im)(im, True) try: im /= afwMath.makeStatistics( im, afwMath.MAX).getValue() except NotImplementedError: pass mos.append( im, label, ds9.GREEN if status == afwMath.SpatialCellCandidate.GOOD else ds9.YELLOW if status == afwMath.SpatialCellCandidate.UNKNOWN else ds9.RED) mos.makeMosaic(frame=8, title="Psf Candidates") # Re-fit until we don't have any candidates with naughty chi^2 values influencing the fit cleanChi2 = False # Any naughty (negative/NAN) chi^2 values? while not cleanChi2: cleanChi2 = True # # First, estimate the PSF # psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # In clipping, allow all candidates to be innocent until proven guilty on this iteration. # Throw out any prima facie guilty candidates (naughty chi^2 values) # for cell in psfCellSet.getCellList(): awfulCandidates = [] for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.PsfCandidateF.cast(cand) cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN ) # until proven guilty rchi2 = cand.getChi2() if not numpy.isfinite(rchi2) or rchi2 <= 0: # Guilty prima facie awfulCandidates.append(cand) cleanChi2 = False self.log.debug("chi^2=%s; id=%s", cand.getChi2(), cand.getSource().getId()) for cand in awfulCandidates: if display: print("Removing bad candidate: id=%d, chi^2=%f" % \ (cand.getSource().getId(), cand.getChi2())) cell.removeCandidate(cand) # # Clip out bad fits based on reduced chi^2 # badCandidates = list() for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates cand = algorithmsLib.PsfCandidateF.cast(cand) rchi2 = cand.getChi2( ) # reduced chi^2 when fitting PSF to candidate assert rchi2 > 0 if rchi2 > self.config.reducedChi2ForPsfCandidates: badCandidates.append(cand) badCandidates.sort(key=lambda x: x.getChi2(), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(numBad), badCandidates): if display: chi2 = c.getChi2() if chi2 > 1e100: chi2 = numpy.nan print("Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2)) c.setStatus(afwMath.SpatialCellCandidate.BAD) # # Clip out bad fits based on spatial fitting. # # This appears to be better at getting rid of sources that have a single dominant kernel component # (other than the zeroth; e.g., a nearby contaminant) because the surrounding sources (which help # set the spatial model) don't contain that kernel component, and so the spatial modeling # downweights the component. # residuals = list() candidates = list() kernel = psf.getKernel() noSpatialKernel = afwMath.cast_LinearCombinationKernel( psf.getKernel()) for cell in psfCellSet.getCellList(): for cand in cell.begin(False): cand = algorithmsLib.PsfCandidateF.cast(cand) candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter()) try: im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight()) except Exception as e: continue fit = algorithmsLib.fitKernelParamsToImage( noSpatialKernel, im, candCenter) params = fit[0] kernels = fit[1] amp = 0.0 for p, k in zip(params, kernels): amp += p * afwMath.cast_FixedKernel(k).getSum() predict = [ kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY()) for k in range(kernel.getNKernelParameters()) ] #print cand.getSource().getId(), [a / amp for a in params], predict residuals.append( [a / amp - p for a, p in zip(params, predict)]) candidates.append(cand) residuals = numpy.array(residuals) for k in range(kernel.getNKernelParameters()): if False: # Straight standard deviation mean = residuals[:, k].mean() rms = residuals[:, k].std() elif False: # Using interquartile range sr = numpy.sort(residuals[:, k]) mean = sr[int(0.5*len(sr))] if len(sr) % 2 else \ 0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1]) rms = 0.74 * (sr[int(0.75 * len(sr))] - sr[int(0.25 * len(sr))]) else: stats = afwMath.makeStatistics( residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP) mean = stats.getValue(afwMath.MEANCLIP) rms = stats.getValue(afwMath.STDEVCLIP) rms = max( 1.0e-4, rms) # Don't trust RMS below this due to numerical issues if display: print("Mean for component %d is %f" % (k, mean)) print("RMS for component %d is %f" % (k, rms)) badCandidates = list() for i, cand in enumerate(candidates): if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject * rms: badCandidates.append(i) badCandidates.sort( key=lambda x: numpy.fabs(residuals[x, k] - mean), reverse=True) numBad = numCandidatesToReject(len(badCandidates), iterNum, self.config.nIterForPsf) for i, c in zip(range(min(len(badCandidates), numBad)), badCandidates): cand = candidates[c] if display: print("Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \ (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k, residuals[badCandidates[i], k], self.config.spatialReject * rms)) cand.setStatus(afwMath.SpatialCellCandidate.BAD) # # Display results # if display and displayIterations: if displayExposure: if iterNum > 0: ds9.erase(frame=frame) maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", size=8, frame=frame, ctype=ds9.YELLOW, ctypeBad=ds9.RED, ctypeUnused=ds9.MAGENTA) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: maUtils.showPsfSpatialCells( exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", size=10, frame=frame, ctype=ds9.YELLOW, ctypeBad=ds9.RED) if displayResiduals: while True: try: maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=5, normalize=normalizeResiduals, showBadCandidates=showBadCandidates, variance=True) except: if not showBadCandidates: showBadCandidates = True continue break if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale('linear', 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) if pause: while True: try: reply = input( "Next iteration? [ynchpqQs] ").strip() except EOFError: reply = "n" reply = reply.split() if reply: reply, args = reply[0], reply[1:] else: reply = "" if reply in ("", "c", "h", "n", "p", "q", "Q", "s", "y"): if reply == "c": pause = False elif reply == "h": print("c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \ "s[ave fileName] y[es]") continue elif reply == "p": import pdb pdb.set_trace() elif reply == "q": display = False elif reply == "Q": sys.exit(1) elif reply == "s": fileName = args.pop(0) if not fileName: print("Please provide a filename") continue print("Saving to %s" % fileName) maUtils.saveSpatialCellSet(psfCellSet, fileName=fileName) continue break else: print("Unrecognised response: %s" % reply, file=sys.stderr) if reply == "n": break # One last time, to take advantage of the last iteration psf, eigenValues, nEigenComponents, fitChi2 = \ self._fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents) # # Display code for debugging # if display and reply != "n": if displayExposure: maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=True, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame) if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell: maUtils.showPsfSpatialCells( exposure, psfCellSet, self.config.nStarPerCellSpatialFit, symb="o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=10, frame=frame) if displayResiduals: maUtils.showPsfCandidates( exposure, psfCellSet, psf=psf, frame=4, normalize=normalizeResiduals, showBadCandidates=showBadCandidates) if displayPsfComponents: maUtils.showPsf(psf, eigenValues, frame=6) if displayPsfMosaic: maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=True) ds9.scale("linear", 0, 1, frame=7) if displayPsfSpatialModel: maUtils.plotPsfSpatialModel( exposure, psf, psfCellSet, showBadCandidates=True, matchKernelAmplitudes=matchKernelAmplitudes, keepPlots=keepMatplotlibPlots) # # Generate some QA information # # Count PSF stars # numGoodStars = 0 numAvailStars = 0 avgX = 0.0 avgY = 0.0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # don't ignore BAD stars numAvailStars += 1 for cand in cell.begin(True): # do ignore BAD stars cand = algorithmsLib.PsfCandidateF.cast(cand) src = cand.getSource() if flagKey is not None: src.set(flagKey, True) avgX += src.getX() avgY += src.getY() numGoodStars += 1 avgX /= numGoodStars avgY /= numGoodStars if metadata is not None: metadata.set("spatialFitChi2", fitChi2) metadata.set("numGoodStars", numGoodStars) metadata.set("numAvailStars", numAvailStars) metadata.set("avgX", avgX) metadata.set("avgY", avgY) psf = algorithmsLib.PcaPsf(psf.getKernel(), afwGeom.Point2D(avgX, avgY)) return psf, psfCellSet