def testReferenceFilter(self): """Test sub-selecting reference objects by flux.""" sourceCat = self.loadSourceCatalog(self.filename) refCat = self.computePosRefCatalog(sourceCat) # Apply source selector to sourceCat, using the astrometry config defaults tempConfig = measAstrom.AstrometryTask.ConfigClass() tempSolver = measAstrom.AstrometryTask(config=tempConfig, refObjLoader=None) sourceSelection = tempSolver.sourceSelector.run(sourceCat) distortedCat = distort.distortList(sourceSelection.sourceCat, distort.linearXDistort) matchPessConfig = measAstrom.MatchPessimisticBTask.ConfigClass() matchPessConfig.maxRefObjects = 150 matchPessConfig.minMatchDistPixels = 5.0 matchPess = measAstrom.MatchPessimisticBTask(config=matchPessConfig) trimmedRefCat = matchPess._filterRefCat(refCat, 'r_flux') self.assertEqual(len(trimmedRefCat), matchPessConfig.maxRefObjects) matchRes = matchPess.matchObjectsToSources( refCat=refCat, sourceCat=distortedCat, wcs=self.distortedWcs, sourceFluxField='slot_ApFlux_instFlux', refFluxField="r_flux", ) self.assertEqual(len(matchRes.matches), matchPessConfig.maxRefObjects - 3)
def getAstrometrySolution(self, loglvl=Log.INFO): astromConfig = measAstrom.AstrometryTask.ConfigClass() astrom = measAstrom.AstrometryTask(config=astromConfig) # use solve instead of run because the exposure has the wrong bbox return astrom.solve( sourceCat=self.srcCat, exposure=self.exposure, )
def testPassingMatcherState(self): """Test that results of the matcher can be propagated to to in subsequent iterations. """ sourceCat = self.loadSourceCatalog(self.filename) refCat = self.computePosRefCatalog(sourceCat) # Apply source selector to sourceCat, using the astrometry config defaults tempConfig = measAstrom.AstrometryTask.ConfigClass() tempSolver = measAstrom.AstrometryTask(config=tempConfig, refObjLoader=None) sourceSelection = tempSolver.sourceSelector.run(sourceCat) distortedCat = distort.distortList(sourceSelection.sourceCat, distort.linearXDistort) sourceCat = distortedCat matchRes = self.MatchPessimisticB.matchObjectsToSources( refCat=refCat, sourceCat=sourceCat, wcs=self.distortedWcs, sourceFluxField='slot_ApFlux_instFlux', refFluxField="r_flux", ) maxShift = matchRes.match_tolerance.maxShift * 300 # Force the matcher to use a different pattern thatn the previous # "iteration". matchTol = measAstrom.MatchTolerancePessimistic( maxMatchDist=matchRes.match_tolerance.maxMatchDist, autoMaxMatchDist=matchRes.match_tolerance.autoMaxMatchDist, maxShift=maxShift, lastMatchedPattern=0, failedPatternList=[0], PPMbObj=matchRes.match_tolerance.PPMbObj, ) matchRes = self.MatchPessimisticB.matchObjectsToSources( refCat=refCat, sourceCat=sourceCat, wcs=self.distortedWcs, sourceFluxField='slot_ApFlux_instFlux', refFluxField="r_flux", match_tolerance=matchTol, ) self.assertEqual(len(matchRes.matches), self.expectedMatches) self.assertLess(matchRes.match_tolerance.maxShift, maxShift) self.assertEqual(matchRes.match_tolerance.lastMatchedPattern, 1) self.assertIsNotNone(matchRes.match_tolerance.maxMatchDist) self.assertIsNotNone(matchRes.match_tolerance.autoMaxMatchDist) self.assertIsNotNone(matchRes.match_tolerance.lastMatchedPattern) self.assertIsNotNone(matchRes.match_tolerance.failedPatternList) self.assertIsNotNone(matchRes.match_tolerance.PPMbObj)
def singleTestInstance(self, filename, distortFunc, doPlot=False): sourceCat = self.loadSourceCatalog(self.filename) refCat = self.computePosRefCatalog(sourceCat) # Apply source selector to sourceCat, using the astrometry config defaults tempConfig = measAstrom.AstrometryTask.ConfigClass() tempConfig.matcher.retarget(measAstrom.MatchOptimisticBTask) tempConfig.sourceSelector["matcher"].excludePixelFlags = False tempSolver = measAstrom.AstrometryTask(config=tempConfig, refObjLoader=None) sourceSelection = tempSolver.sourceSelector.run(sourceCat) distortedCat = distort.distortList(sourceSelection.sourceCat, distortFunc) if doPlot: import matplotlib.pyplot as plt undistorted = [ self.wcs.skyToPixel( self.distortedWcs.pixelToSky(ss.getCentroid())) for ss in distortedCat ] refs = [self.wcs.skyToPixel(ss.getCoord()) for ss in refCat] def plot(catalog, symbol): plt.plot([ss.getX() for ss in catalog], [ss.getY() for ss in catalog], symbol) # plot(sourceCat, 'k+') # Original positions: black + plot(distortedCat, 'b+') # Distorted positions: blue + plot(undistorted, 'g+') # Undistorted positions: green + plot(refs, 'rx') # Reference catalog: red x # The green + should overlap with the red x, because that's how matchOptimisticB does it. # The black + happens to overlap with those also, but that's beside the point. plt.show() sourceCat = distortedCat matchRes = self.matchOptimisticB.matchObjectsToSources( refCat=refCat, sourceCat=sourceCat, wcs=self.distortedWcs, sourceFluxField='slot_ApFlux_instFlux', refFluxField="r_flux", ) matches = matchRes.matches if doPlot: measAstrom.plotAstrometry(matches=matches, refCat=refCat, sourceCat=sourceCat) self.assertEqual(len(matches), 183) refCoordKey = afwTable.CoordKey(refCat.schema["coord"]) srcCoordKey = afwTable.CoordKey(sourceCat.schema["coord"]) refCentroidKey = afwTable.Point2DKey(refCat.getSchema()["centroid"]) maxDistErr = 0 * lsst.geom.radians for refObj, source, distRad in matches: sourceCoord = source.get(srcCoordKey) refCoord = refObj.get(refCoordKey) predDist = sourceCoord.separation(refCoord) distErr = abs(predDist - distRad * lsst.geom.radians) maxDistErr = max(distErr, maxDistErr) if refObj.getId() != source.getId(): refCentroid = refObj.get(refCentroidKey) sourceCentroid = source.getCentroid() radius = math.hypot(*(refCentroid - sourceCentroid)) self.fail( "ID mismatch: %s at %s != %s at %s; error = %0.1f pix" % (refObj.getId(), refCentroid, source.getId(), sourceCentroid, radius)) self.assertLess(maxDistErr.asArcseconds(), 1e-7)
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = long(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution ctr = afwGeom.Box2D(exposure.getBBox()).getCenter() psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr)) scienceSigmaOrig = psfAttr.computeGaussianWidth( psfAttr.ADAPTIVE_MOMENT) # sigma of PSF of template image before warping ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter() psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr)) templateSigma = psfAttr.computeGaussianWidth( psfAttr.ADAPTIVE_MOMENT) # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel( ).getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn( "Src product does not exist; running detection, measurement, selection" ) # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma=scienceSigmaPost, doSmooth=not self.doPreConvolve, idFactory=idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len( makeKernelBasisList( self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch( exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources matches = afwTable.matchRaDec(templateSources, selectSources, 1.0 * afwGeom.arcseconds, False) else: raise RuntimeError( "doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars( exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [ k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize ] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info( "Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId(): x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable( ).getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable( ).getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [ m.first.get(refCoordKey) for m in wcsResults.matches ] residuals = [ m.first.get(refCoordKey).getOffsetFrom( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [ m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] colors = numpy.array([ -2.5 * numpy.log10(srcToMatch[x].get("g")) + 2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys() ]) dlong = numpy.array([ r[0].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) dlat = numpy.array([ r[1].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) idx1 = numpy.where( colors < self.sourceSelector.config.grMin) idx2 = numpy.where( (colors >= self.sourceSelector.config.grMin) & (colors <= self.sourceSelector.config.grMax)) idx3 = numpy.where( colors > self.sourceSelector.config.grMax) rms1Long = IqrToSigma * (numpy.percentile( dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)) rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) - numpy.percentile(dlat[idx1], 25)) rms2Long = IqrToSigma * (numpy.percentile( dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)) rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) - numpy.percentile(dlat[idx2], 25)) rms3Long = IqrToSigma * (numpy.percentile( dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)) rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) - numpy.percentile(dlat[idx3], 25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info( "Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference #Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Computing diffim PSF") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run( exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False. if self.config.doDecorrelation and self.config.doSubtract: decorrResult = self.decorrelate.run(exposure, templateExposure, subtractedExposure, subtractRes.psfMatchingKernel) subtractedExposure = decorrResult.correctedExposure if self.config.doDetection: self.log.info("Running diaSource detection") # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if not self.config.doDipoleFitting: self.measurement.run(diaSources, subtractedExposure) else: if self.config.doSubtract: self.measurement.run(diaSources, subtractedExposure, exposure, subtractRes.matchedExposure) else: self.measurement.run(diaSources, subtractedExposure, exposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs( ).pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn( "Src product does not exist; cannot match with diaSources" ) srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = measAstrom.AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = measAstrom.AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn( "No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info( "Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \ refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if srcMatchDict.has_key(sid): diaSource.set("srcMatchId", srcMatchDict[sid]) if refMatchDict.has_key(sid): diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cast_KernelCandidateF(cand)) # Get basis list to build control sample kernels basisList = afwMath.cast_LinearCombinationKernel( kernelCandList[0].getKernel( KernelCandidateF.ORIG)).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )