def run(self, dataRef): """!Compute a few statistics on the image plane of an exposure @param dataRef: data reference for a calibrated science exposure ("calexp") @return a pipeBase Struct containing: - mean: mean of image plane - meanErr: uncertainty in mean - stdDev: standard deviation of image plane - stdDevErr: uncertainty in standard deviation """ self.log.info("Processing data ID %s" % (dataRef.dataId, )) if self.config.doFail: raise pipeBase.TaskError( "Raising TaskError by request (config.doFail=True)") # Unpersist the raw exposure pointed to by the data reference rawExp = dataRef.get("raw") maskedImage = rawExp.getMaskedImage() # Support extra debug output. # - import lsstDebug display = lsstDebug.Info(__name__).display if display: frame = 1 mtv(rawExp, frame=frame, title="exposure") # return the pipe_base Struct that is returned by self.stats.run return self.stats.run(maskedImage)
def runDataRef(self, dataRefList, camera, butler, tract, debug, diagDir=".", diagnostics=False, snapshots=False, numCoresForReadSource=1, readTimeout=9999, verbose=False): self.log.info("Running self-calibration for tract %d" % tract) skyMap = butler.get("deepCoadd_skyMap", immediate=True) tractInfo = skyMap[tract] filters = set(dataRef.dataId['filter'] for dataRef in dataRefList) if len(filters) != 1: self.log.warn("There are %d filters in input frames: %s" % (len(filters), ", ".join(filters))) if not self.config.allowMixedFilters: raise pipeBase.TaskError("Multiple filters found: %s" % (filters,)) if self.config.doColorTerms and self.config.photoCatName: filterName = sorted(filters)[0] self.log.info("Using color terms for filter %s" % filterName) ct = self.config.colorterms.getColorterm(filterName, self.config.photoCatName) self.log.info("color term: " + str(ct)) elif self.config.doColorTerms: ct = None self.log.warn("Cannot apply color term: reference catalog not specified") else: ct = None self.log.info("Not applying color term") return self.run(dataRefList, tractInfo, ct, debug, diagDir, diagnostics, snapshots, numCoresForReadSource, readTimeout, verbose)
def runDataRef(self, dataRef): if self.config.doFail: raise pipeBase.TaskError( "Failed by request: config.doFail is true") self.dataRefList.append(dataRef) self.numProcessed += 1 self.metadata["numProcessed"] = self.numProcessed return pipeBase.Struct(numProcessed=self.numProcessed, )
def fetchInPatches(self, butler, exposure, tract, patchList): """! Get the reference catalogs from a given tract,patchlist @param[in] butler A Butler used to get the reference catalogs @param[in] exposure A deepDiff_exposure on which to run the measurements @param[in] tract The tract @param[in] patchList A list of patches that need to be checked @return Combined SourceCatalog from all the patches """ dataset = f"{self.config.coaddName}Diff_diaObject" catalog = None for patch in patchList: dataId = {'tract': tract.getId(), 'patch': "%d,%d" % patch.getIndex()} self.log.info("Getting references in %s" % (dataId,)) if not butler.datasetExists(dataset, dataId): if self.config.skipMissing: self.log.info("Could not find %s for dataset %s" % (dataId, dataset)) continue raise pipeBase.TaskError("Reference %s doesn't exist" % (dataId,)) new_catalog = butler.get(dataset, dataId, immediate=True) patchBox = geom.Box2D(patch.getInnerBBox()) tractBox = tract.getInnerSkyPolygon() tractWcs = tract.getWcs() expBox = geom.Box2D(exposure.getBBox()) expWcs = exposure.getWcs() # only use objects that overlap inner patch bounding box and overlap exposure validPatch = np.array([ patchBox.contains(tractWcs.skyToPixel(s.getCoord())) for s in new_catalog]) # There doesn't seem to be a inner bounding box so I have to use the sphgeom stuff validTract = np.array( [tractBox.contains(sphgeom.UnitVector3d(sphgeom.LonLat.fromRadians(s.getRa().asRadians(), s.getDec().asRadians()))) for s in new_catalog]) validExposure = np.array([ expBox.contains(expWcs.skyToPixel(s.getCoord())) for s in new_catalog]) if validPatch.size == 0 or validExposure.size == 0 or validTract.size == 0: self.log.debug("No valid sources %s for dataset %s" % (dataId, dataset)) continue if catalog is None: catalog = new_catalog[validPatch & validExposure & validTract] else: catalog.extend(new_catalog[validPatch & validExposure & validTract]) return catalog
def run(self, exposure, sensorRef, templateIdList): """Return a calexp exposure with based on input sensorRef. Construct a dataId based on the sensorRef.dataId combined with the specifications from the first dataId in templateIdList Parameters ---------- exposure : `lsst.afw.image.Exposure` exposure (unused) sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` Data reference of the calexp(s) to subtract from. templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` Data reference of the template calexp to be subtraced. Can be incomplete, fields are initialized from `sensorRef`. If there are multiple items, only the first one is used. Returns ------- result : `struct` return a pipeBase.Struct: - ``exposure`` : a template calexp - ``sources`` : source catalog measured on the template """ if len(templateIdList) == 0: raise RuntimeError("No template data reference supplied.") if len(templateIdList) > 1: self.log.warning( "Multiple template data references supplied. Using the first one only." ) templateId = sensorRef.dataId.copy() templateId.update(templateIdList[0]) self.log.info("Fetching calexp (%s) as template.", templateId) butler = sensorRef.getButler() template = butler.get(datasetType="calexp", dataId=templateId) if self.config.doAddCalexpBackground: templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) mi = template.getMaskedImage() mi += templateBg.getImage() if not template.hasPsf(): raise pipeBase.TaskError("Template has no psf") templateSources = butler.get(datasetType="src", dataId=templateId) return pipeBase.Struct(exposure=template, sources=templateSources)
def run(self, exposure, sensorRef, templateIdList): """!Return a calexp exposure with same ccd as input sensorRef. \param[in] exposure -- exposure (unused) \param[in] sensorRef -- a Butler data reference \param[in] templateIdList -- list of data ids. Only visit used. \return a pipeBase.Struct - exposure: a template calexp - sources: source catalog measured on the template """ #if len(templateIdList): # self.log.warn("Multiple template visits supplied. Getting template from first visit: %s" % # (templateIdList[0]['visit'])) #print "refList",templateIdList.refList #print "dataId", templateIdList.dataId print "....." #calexps = templateIdList.getButler().get("calexp") for elem in templateIdList: print elem elem["visit"] = int(elem["visit"]) elem["ccd"] = int(elem["ccd"]) # print elem.dataId["visit"] print "....." templateId = templateIdList[0] # templateId = templateIdList[0].dataId #dataRefList = templateIdList.idList #templateId["visit"] = dataRefList[0]['visit'] #templateId = templateIdList.refList self.log.info("Fetching calexp (%s) as template." % (templateId)) #print templateId butler = sensorRef.getButler() template = butler.get(datasetType="calexp", dataId=templateId) if self.config.doAddCalexpBackground: templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) mi = template.getMaskedImage() mi += templateBg.getImage() if not template.hasPsf(): raise pipeBase.TaskError("Template has no psf") templateSources = butler.get(datasetType="src", dataId=templateId) return pipeBase.Struct(exposure=template, sources=templateSources)
def run(self, exposure, sensorRef, templateIdList): """!Return a calexp exposure with based on input sensorRef. Construct a dataId based on the sensorRef.dataId combined with the specifications from the first dataId in templateIdList \param[in] exposure -- exposure (unused) \param[in] sensorRef -- a Butler data reference \param[in] templateIdList -- list of data ids, which should contain a single item. If there are multiple items, only the first is used. \return a pipeBase.Struct - exposure: a template calexp - sources: source catalog measured on the template """ if len(templateIdList) == 0: raise RuntimeError( "No template supplied! Please supply a template visit id.") if len(templateIdList) > 1: self.log.warn( "Multiple template visits supplied. Getting template from first visit: %s" % (templateIdList[0]['visit'])) templateId = sensorRef.dataId.copy() templateId.update(templateIdList[0]) self.log.info("Fetching calexp (%s) as template." % (templateId)) butler = sensorRef.getButler() template = butler.get(datasetType="calexp", dataId=templateId) if self.config.doAddCalexpBackground: templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) mi = template.getMaskedImage() mi += templateBg.getImage() if not template.hasPsf(): raise pipeBase.TaskError("Template has no psf") templateSources = butler.get(datasetType="src", dataId=templateId) return pipeBase.Struct(exposure=template, sources=templateSources)
def fetchInPatches(self, dataRef, patchList): """! Copied from CoaddSrcReferencesTask and modified to allow loading deepDiff_diaObjects. The given dataRef must include the tract in its dataId. """ dataset = "deepDiff_diaObject" tract = dataRef.dataId["tract"] butler = dataRef.butlerSubset.butler for patch in patchList: dataId = {'tract': tract, 'patch': "%d,%d" % patch.getIndex()} if not butler.datasetExists(dataset, dataId): if self.config.skipMissing: continue raise pipeBase.TaskError("Reference %s doesn't exist" % (dataId, )) self.log.info("Getting references in %s" % (dataId, )) catalog = butler.get(dataset, dataId, immediate=True) for source in catalog: yield source
def selectRefExposure(self, expRefList, imageScalerList, expDatasetType): """Find best exposure to use as the reference exposure Calculate an appropriate reference exposure by minimizing a cost function that penalizes high variance, high background level, and low coverage. Use the following config parameters: - bestRefWeightCoverage - bestRefWeightVariance - bestRefWeightLevel @param[in] expRefList: list of data references to exposures. Retrieves dataset type specified by expDatasetType. If an exposure is not found, it is skipped with a warning. @param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler); must be the same length as expRefList @param[in] expDatasetType: dataset type of exposure: e.g. 'goodSeeingCoadd_tempExp' @return: index of best exposure @raise pipeBase.TaskError if none of the exposures in expRefList are found. """ self.log.info("Calculating best reference visit") varList = [] meanBkgdLevelList = [] coverageList = [] if len(expRefList) != len(imageScalerList): raise RuntimeError( "len(expRefList) = %s != %s = len(imageScalerList)" % (len(expRefList), len(imageScalerList))) for expRef, imageScaler in zip(expRefList, imageScalerList): exposure = expRef.get(expDatasetType, immediate=True) maskedImage = exposure.getMaskedImage() if imageScaler is not None: try: imageScaler.scaleMaskedImage(maskedImage) except: # need to put a place holder in Arr varList.append(numpy.nan) meanBkgdLevelList.append(numpy.nan) coverageList.append(numpy.nan) continue statObjIm = afwMath.makeStatistics( maskedImage.getImage(), maskedImage.getMask(), afwMath.MEAN | afwMath.NPOINT | afwMath.VARIANCE, self.sctrl) meanVar, meanVarErr = statObjIm.getResult(afwMath.VARIANCE) meanBkgdLevel, meanBkgdLevelErr = statObjIm.getResult(afwMath.MEAN) npoints, npointsErr = statObjIm.getResult(afwMath.NPOINT) varList.append(meanVar) meanBkgdLevelList.append(meanBkgdLevel) coverageList.append(npoints) if not coverageList: raise pipeBase.TaskError( "None of the candidate %s exist; cannot select best reference exposure" % (expDatasetType, )) # Normalize metrics to range from 0 to 1 varArr = numpy.array(varList) / numpy.nanmax(varList) meanBkgdLevelArr = numpy.array(meanBkgdLevelList) / numpy.nanmax( meanBkgdLevelList) coverageArr = numpy.nanmin(coverageList) / numpy.array(coverageList) costFunctionArr = self.config.bestRefWeightVariance * varArr costFunctionArr += self.config.bestRefWeightLevel * meanBkgdLevelArr costFunctionArr += self.config.bestRefWeightCoverage * coverageArr return numpy.nanargmin(costFunctionArr)
def run(self, expRefList, expDatasetType, imageScalerList=None, refExpDataRef=None, refImageScaler=None): """Match the backgrounds of a list of coadd temp exposures to a reference coadd temp exposure. Choose a refExpDataRef automatically if none supplied. @param[in] expRefList: list of data references to science exposures to be background-matched; all exposures must exist. @param[in] expDatasetType: dataset type of exposures, e.g. 'goodSeeingCoadd_tempExp' @param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler); if None then the images are not scaled @param[in] refExpDataRef: data reference for the reference exposure. If None, then this task selects the best exposures from expRefList. if not None then must be one of the exposures in expRefList. @param[in] refImageScaler: image scaler for reference image; ignored if refExpDataRef is None, else scaling is not performed if None @return: a pipBase.Struct containing these fields: - backgroundInfoList: a list of pipeBase.Struct, one per exposure in expRefList, each of which contains these fields: - isReference: this is the reference exposure (only one returned Struct will contain True for this value, unless the ref exposure is listed multiple times) - backgroundModel: differential background model (afw.Math.Background or afw.Math.Approximate). Add this to the science exposure to match the reference exposure. - fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)). - matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2); should be comparable to difference image's mean variance. - diffImVar: the mean variance of the difference image. All fields except isReference will be None if isReference True or the fit failed. @warning: all exposures must exist on disk """ numExp = len(expRefList) if numExp < 1: raise pipeBase.TaskError("No exposures to match") if expDatasetType is None: raise pipeBase.TaskError("Must specify expDatasetType") if imageScalerList is None: self.log.info( "imageScalerList is None; no scaling will be performed") imageScalerList = [None] * numExp if len(expRefList) != len(imageScalerList): raise RuntimeError( "len(expRefList) = %s != %s = len(imageScalerList)" % (len(expRefList), len(imageScalerList))) refInd = None if refExpDataRef is None: # select the best reference exposure from expRefList refInd = self.selectRefExposure( expRefList=expRefList, imageScalerList=imageScalerList, expDatasetType=expDatasetType, ) refExpDataRef = expRefList[refInd] refImageScaler = imageScalerList[refInd] # refIndSet is the index of all exposures in expDataList that match the reference. # It is used to avoid background-matching an exposure to itself. It is a list # because it is possible (though unlikely) that expDataList will contain duplicates. expKeyList = refExpDataRef.butlerSubset.butler.getKeys(expDatasetType) refMatcher = DataRefMatcher(refExpDataRef.butlerSubset.butler, expDatasetType) refIndSet = set( refMatcher.matchList(ref0=refExpDataRef, refList=expRefList)) if refInd is not None and refInd not in refIndSet: raise RuntimeError( "Internal error: selected reference %s not found in expRefList" ) refExposure = refExpDataRef.get(expDatasetType, immediate=True) if refImageScaler is not None: refMI = refExposure.getMaskedImage() refImageScaler.scaleMaskedImage(refMI) debugIdKeyList = tuple(set(expKeyList) - set(['tract', 'patch'])) self.log.info("Matching %d Exposures" % (numExp)) backgroundInfoList = [] for ind, (toMatchRef, imageScaler) in enumerate(zip(expRefList, imageScalerList)): if ind in refIndSet: backgroundInfoStruct = pipeBase.Struct( isReference=True, backgroundModel=None, fitRMS=0.0, matchedMSE=None, diffImVar=None, ) else: self.log.info("Matching background of %s to %s" % (toMatchRef.dataId, refExpDataRef.dataId)) try: toMatchExposure = toMatchRef.get(expDatasetType, immediate=True) if imageScaler is not None: toMatchMI = toMatchExposure.getMaskedImage() imageScaler.scaleMaskedImage(toMatchMI) # store a string specifying the visit to label debug plot self.debugDataIdString = ''.join( [str(toMatchRef.dataId[vk]) for vk in debugIdKeyList]) backgroundInfoStruct = self.matchBackgrounds( refExposure=refExposure, sciExposure=toMatchExposure, ) backgroundInfoStruct.isReference = False except Exception as e: self.log.warn("Failed to fit background %s: %s" % (toMatchRef.dataId, e)) backgroundInfoStruct = pipeBase.Struct( isReference=False, backgroundModel=None, fitRMS=None, matchedMSE=None, diffImVar=None, ) backgroundInfoList.append(backgroundInfoStruct) return pipeBase.Struct(backgroundInfoList=backgroundInfoList)
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = int(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius() # sigma of PSF of template image before warping templateSigma = templateExposure.getPsf().computeShape( ).getDeterminantRadius() # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel( ).getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn( "Src product does not exist; running detection, measurement, selection" ) # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma=scienceSigmaPost, doSmooth=not self.doPreConvolve, idFactory=idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len( makeKernelBasisList( self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch( exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(templateSources, selectSources, 1.0 * afwGeom.arcseconds, mc) else: raise RuntimeError( "doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars( exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [ k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize ] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info( "Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId(): x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable( ).getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable( ).getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [ m.first.get(refCoordKey) for m in wcsResults.matches ] residuals = [ m.first.get(refCoordKey).getOffsetFrom( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [ m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] colors = numpy.array([ -2.5 * numpy.log10(srcToMatch[x].get("g")) + 2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys() ]) dlong = numpy.array([ r[0].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) dlat = numpy.array([ r[1].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) idx1 = numpy.where( colors < self.sourceSelector.config.grMin) idx2 = numpy.where( (colors >= self.sourceSelector.config.grMin) & (colors <= self.sourceSelector.config.grMax)) idx3 = numpy.where( colors > self.sourceSelector.config.grMax) rms1Long = IqrToSigma * \ (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25)) rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) - numpy.percentile(dlat[idx1], 25)) rms2Long = IqrToSigma * \ (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25)) rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) - numpy.percentile(dlat[idx2], 25)) rms3Long = IqrToSigma * \ (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25)) rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) - numpy.percentile(dlat[idx3], 25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info( "Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference # Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Computing diffim PSF") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run( exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False. if self.config.doDecorrelation and self.config.doSubtract: decorrResult = self.decorrelate.run(exposure, templateExposure, subtractedExposure, subtractRes.psfMatchingKernel) subtractedExposure = decorrResult.correctedExposure if self.config.doDetection: self.log.info("Running diaSource detection") # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if not self.config.doDipoleFitting: self.measurement.run(diaSources, subtractedExposure) else: if self.config.doSubtract: self.measurement.run(diaSources, subtractedExposure, exposure, subtractRes.matchedExposure) else: self.measurement.run(diaSources, subtractedExposure, exposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs( ).pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn( "Src product does not exist; cannot match with diaSources" ) srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn( "No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info( "Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if sid in srcMatchDict: diaSource.set("srcMatchId", srcMatchDict[sid]) if sid in refMatchDict: diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cand) # Get basis list to build control sample kernels basisList = kernelCandList[0].getKernel( KernelCandidateF.ORIG).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )
def matchObjectsToSources(self, refCat, sourceCat, wcs, sourceFluxField, refFluxField, match_tolerance=None): """Match sources to position reference stars refCat : `lsst.afw.table.SimpleCatalog` catalog of reference objects that overlap the exposure; reads fields for: - coord - the specified flux field sourceCat : `lsst.afw.table.SourceCatalog` Catalog of sources found on an exposure. This should already be down-selected to "good"/"usable" sources in the calling Task. wcs : `lsst.afw.geom.SkyWcs` estimated WCS sourceFluxField: `str` field of sourceCat to use for flux refFluxField : `str` field of refCat to use for flux match_tolerance : `lsst.meas.astrom.MatchTolerancePessimistic` is a MatchTolerance class object or `None`. This this class is used to communicate state between AstrometryTask and MatcherTask. AstrometryTask will also set the MatchTolerance class variable maxMatchDist based on the scatter AstrometryTask has found after fitting for the wcs. Returns ------- result : `lsst.pipe.base.Struct` Result struct with components: - ``matches`` : source to reference matches found (`list` of `lsst.afw.table.ReferenceMatch`) - ``usableSourceCat`` : a catalog of sources potentially usable for matching and WCS fitting (`lsst.afw.table.SourceCatalog`). - ``match_tolerance`` : a MatchTolerance object containing the resulting state variables from the match (`lsst.meas.astrom.MatchTolerancePessimistic`). """ import lsstDebug debug = lsstDebug.Info(__name__) # If we get an empty tolerance struct create the variables we need for # this matcher. if match_tolerance is None: match_tolerance = MatchTolerancePessimistic() # Make a name alias here for consistency with older code, and to make # it clear that this is a good/usable (cleaned) source catalog. goodSourceCat = sourceCat numUsableSources = len(goodSourceCat) if len(goodSourceCat) == 0: raise pipeBase.TaskError("No sources are good") minMatchedPairs = min(self.config.minMatchedPairs, int(self.config.minFracMatchedPairs * min([len(refCat), len(goodSourceCat)]))) if len(refCat) > self.config.maxRefObjects: self.log.warn( "WARNING: Reference catalog larger that maximum allowed. " "Trimming to %i" % self.config.maxRefObjects) trimmedRefCat = self._filterRefCat(refCat, refFluxField) else: trimmedRefCat = refCat doMatchReturn = self._doMatch( refCat=trimmedRefCat, sourceCat=goodSourceCat, wcs=wcs, refFluxField=refFluxField, numUsableSources=numUsableSources, minMatchedPairs=minMatchedPairs, match_tolerance=match_tolerance, sourceFluxField=sourceFluxField, verbose=debug.verbose, ) matches = doMatchReturn.matches match_tolerance = doMatchReturn.match_tolerance if len(matches) == 0: raise RuntimeError("Unable to match sources") self.log.info("Matched %d sources" % len(matches)) if len(matches) < minMatchedPairs: self.log.warn("Number of matches is smaller than request") return pipeBase.Struct( matches=matches, usableSourceCat=goodSourceCat, match_tolerance=match_tolerance, )
def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True): """!Detect footprints. \param exposure Exposure to process; DETECTED{,_NEGATIVE} mask plane will be set in-place. \param doSmooth if True, smooth the image before detection using a Gaussian of width sigma \param sigma sigma of PSF (pixels); used for smoothing and to grow detections; if None then measure the sigma of the PSF of the exposure \param clearMask Clear both DETECTED and DETECTED_NEGATIVE planes before running detection \return a lsst.pipe.base.Struct with fields: - positive: lsst.afw.detection.FootprintSet with positive polarity footprints (may be None) - negative: lsst.afw.detection.FootprintSet with negative polarity footprints (may be None) - numPos: number of footprints in positive or 0 if detection polarity was negative - numNeg: number of footprints in negative or 0 if detection polarity was positive - background: re-estimated background. None if reEstimateBackground==False \throws lsst.pipe.base.TaskError if sigma=None and the exposure has no PSF """ try: import lsstDebug display = lsstDebug.Info(__name__).display except ImportError: try: display except NameError: display = False if exposure is None: raise RuntimeError("No exposure for detection") maskedImage = exposure.getMaskedImage() region = maskedImage.getBBox() if clearMask: mask = maskedImage.getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) del mask if self.config.doTempLocalBackground: tempBgRes = self.tempLocalBackground.run(maskedImage) tempLocalBkgdImage = tempBgRes.background.getImage() if sigma is None: psf = exposure.getPsf() if psf is None: raise pipeBase.TaskError( "exposure has no PSF; must specify sigma") shape = psf.computeShape() sigma = shape.getDeterminantRadius() self.metadata.set("sigma", sigma) self.metadata.set("doSmooth", doSmooth) if not doSmooth: convolvedImage = maskedImage.Factory(maskedImage) middle = convolvedImage else: # smooth using a Gaussian (which is separate, hence fast) of width sigma # make a SingleGaussian (separable) kernel with the 'sigma' psf = exposure.getPsf() kWidth = (int(sigma * 7 + 0.5) // 2) * 2 + 1 # make sure it is odd self.metadata.set("smoothingKernelWidth", kWidth) gaussFunc = afwMath.GaussianFunction1D(sigma) gaussKernel = afwMath.SeparableKernel(kWidth, kWidth, gaussFunc, gaussFunc) convolvedImage = maskedImage.Factory(maskedImage.getBBox()) afwMath.convolve(convolvedImage, maskedImage, gaussKernel, afwMath.ConvolutionControl()) # # Only search psf-smooth part of frame # goodBBox = gaussKernel.shrinkBBox(convolvedImage.getBBox()) middle = convolvedImage.Factory(convolvedImage, goodBBox, afwImage.PARENT, False) # # Mark the parts of the image outside goodBBox as EDGE # self.setEdgeBits(maskedImage, goodBBox, maskedImage.getMask().getPlaneBitMask("EDGE")) fpSets = pipeBase.Struct(positive=None, negative=None) if self.config.thresholdPolarity != "negative": fpSets.positive = self.thresholdImage(middle, "positive") if self.config.reEstimateBackground or self.config.thresholdPolarity != "positive": fpSets.negative = self.thresholdImage(middle, "negative") for polarity, maskName in (("positive", "DETECTED"), ("negative", "DETECTED_NEGATIVE")): fpSet = getattr(fpSets, polarity) if fpSet is None: continue fpSet.setRegion(region) if self.config.nSigmaToGrow > 0: nGrow = int((self.config.nSigmaToGrow * sigma) + 0.5) self.metadata.set("nGrow", nGrow) fpSet = afwDet.FootprintSet(fpSet, nGrow, self.config.isotropicGrow) fpSet.setMask(maskedImage.getMask(), maskName) if not self.config.returnOriginalFootprints: setattr(fpSets, polarity, fpSet) fpSets.numPos = len(fpSets.positive.getFootprints() ) if fpSets.positive is not None else 0 fpSets.numNeg = len(fpSets.negative.getFootprints() ) if fpSets.negative is not None else 0 if self.config.thresholdPolarity != "negative": self.log.log( self.log.INFO, "Detected %d positive sources to %g sigma." % (fpSets.numPos, self.config.thresholdValue * self.config.includeThresholdMultiplier)) if self.config.doTempLocalBackground: maskedImage += tempLocalBkgdImage fpSets.background = None if self.config.reEstimateBackground: mi = exposure.getMaskedImage() bkgd = self.background.fitBackground(mi) if self.config.adjustBackground: self.log.log( self.log.WARN, "Fiddling the background by %g" % self.config.adjustBackground) bkgd += self.config.adjustBackground fpSets.background = bkgd self.log.log( self.log.INFO, "Resubtracting the background after object detection") mi -= bkgd.getImageF() del mi if self.config.thresholdPolarity == "positive": if self.config.reEstimateBackground: mask = maskedImage.getMask() mask &= ~mask.getPlaneBitMask("DETECTED_NEGATIVE") del mask fpSets.negative = None else: self.log.log( self.log.INFO, "Detected %d negative sources to %g %s" % (fpSets.numNeg, self.config.thresholdValue, ("DN" if self.config.thresholdType == "value" else "sigma"))) if display: ds9.mtv(exposure, frame=0, title="detection") x0, y0 = exposure.getXY0() def plotPeaks(fps, ctype): if fps is None: return with ds9.Buffering(): for fp in fps.getFootprints(): for pp in fp.getPeaks(): ds9.dot("+", pp.getFx() - x0, pp.getFy() - y0, ctype=ctype) plotPeaks(fpSets.positive, "yellow") plotPeaks(fpSets.negative, "red") if convolvedImage and display and display > 1: ds9.mtv(convolvedImage, frame=1, title="PSF smoothed") return fpSets
class SourceDetectionTask(pipeBase.Task): """ Detect positive and negative sources on an exposure and return a new SourceCatalog. """ ConfigClass = SourceDetectionConfig _DefaultName = "sourceDetection" def __init__(self, schema=None, **kwds): """Create the detection task. Most arguments are simply passed onto pipe_base.Task. If schema is not None, it will be used to register a 'flags.negative' flag field that will be set for negative detections. """ pipeBase.Task.__init__(self, **kwds) if schema is not None: self.negativeFlagKey = schema.addField( "flags.negative", type="Flag", doc="set if source was detected as significantly negative") else: if self.config.thresholdPolarity == "both": self.log.log(self.log.WARN, "Detection polarity set to 'both', but no flag will be "\ "set to distinguish between positive and negative detections") self.negativeFlagKey = None @pipeBase.timeMethod def makeSourceCatalog(self, table, exposure, doSmooth=True, sigma=None, clearMask=True): """Run source detection and create a SourceCatalog. To avoid dealing with sources and tables, use detectFootprints() to just get the FootprintSets. @param table lsst.afw.table.SourceTable object that will be used to created the SourceCatalog. @param exposure Exposure to process; DETECTED mask plane will be set in-place. @param doSmooth if True, smooth the image before detection using a Gaussian of width sigma @param sigma sigma of PSF (pixels); used for smoothing and to grow detections; if None then measure the sigma of the PSF of the exposure @param clearMask Clear DETECTED{,_NEGATIVE} planes before running detection @return a Struct with: sources -- an lsst.afw.table.SourceCatalog object fpSets --- Struct returned by detectFootprints @raise pipe_base TaskError if sigma=None, doSmooth=True and the exposure has no PSF """ if self.negativeFlagKey is not None and self.negativeFlagKey not in table.getSchema( ): raise ValueError("Table has incorrect Schema") fpSets = self.detectFootprints(exposure=exposure, doSmooth=doSmooth, sigma=sigma, clearMask=clearMask) sources = afwTable.SourceCatalog(table) table.preallocate(fpSets.numPos + fpSets.numNeg) # not required, but nice if fpSets.negative: fpSets.negative.makeSources(sources) if self.negativeFlagKey: for record in sources: record.set(self.negativeFlagKey, True) if fpSets.positive: fpSets.positive.makeSources(sources) return pipeBase.Struct(sources=sources, fpSets=fpSets) @pipeBase.timeMethod def detectFootprints(self, exposure, doSmooth=True, sigma=None, clearMask=True): """Detect footprints. @param exposure Exposure to process; DETECTED{,_NEGATIVE} mask plane will be set in-place. @param doSmooth if True, smooth the image before detection using a Gaussian of width sigma @param sigma sigma of PSF (pixels); used for smoothing and to grow detections; if None then measure the sigma of the PSF of the exposure @param clearMask Clear both DETECTED and DETECTED_NEGATIVE planes before running detection @return a lsst.pipe.base.Struct with fields: - positive: lsst.afw.detection.FootprintSet with positive polarity footprints (may be None) - negative: lsst.afw.detection.FootprintSet with negative polarity footprints (may be None) - numPos: number of footprints in positive or 0 if detection polarity was negative - numNeg: number of footprints in negative or 0 if detection polarity was positive - background: re-estimated background. None if reEstimateBackground==False @raise pipe_base TaskError if sigma=None and the exposure has no PSF """ try: import lsstDebug display = lsstDebug.Info(__name__).display except ImportError, e: try: display except NameError: display = False if exposure is None: raise RuntimeException("No exposure for detection") maskedImage = exposure.getMaskedImage() region = maskedImage.getBBox(afwImage.PARENT) if clearMask: mask = maskedImage.getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) del mask if sigma is None: psf = exposure.getPsf() if psf is None: raise pipeBase.TaskError( "exposure has no PSF; must specify sigma") shape = psf.computeShape() sigma = shape.getDeterminantRadius() self.metadata.set("sigma", sigma) self.metadata.set("doSmooth", doSmooth) if not doSmooth: convolvedImage = maskedImage.Factory(maskedImage) middle = convolvedImage else: # smooth using a Gaussian (which is separate, hence fast) of width sigma # make a SingleGaussian (separable) kernel with the 'sigma' psf = exposure.getPsf() kWidth = (int(sigma * 7 + 0.5) / 2) * 2 + 1 # make sure it is odd self.metadata.set("smoothingKernelWidth", kWidth) gaussFunc = afwMath.GaussianFunction1D(sigma) gaussKernel = afwMath.SeparableKernel(kWidth, kWidth, gaussFunc, gaussFunc) convolvedImage = maskedImage.Factory( maskedImage.getBBox(afwImage.PARENT)) afwMath.convolve(convolvedImage, maskedImage, gaussKernel, afwMath.ConvolutionControl()) # # Only search psf-smooth part of frame # goodBBox = gaussKernel.shrinkBBox( convolvedImage.getBBox(afwImage.PARENT)) middle = convolvedImage.Factory(convolvedImage, goodBBox, afwImage.PARENT, False) # # Mark the parts of the image outside goodBBox as EDGE # self.setEdgeBits(maskedImage, goodBBox, maskedImage.getMask().getPlaneBitMask("EDGE")) fpSets = pipeBase.Struct(positive=None, negative=None) if self.config.thresholdPolarity != "negative": fpSets.positive = self.thresholdImage(middle, "positive") if self.config.reEstimateBackground or self.config.thresholdPolarity != "positive": fpSets.negative = self.thresholdImage(middle, "negative") for polarity, maskName in (("positive", "DETECTED"), ("negative", "DETECTED_NEGATIVE")): fpSet = getattr(fpSets, polarity) if fpSet is None: continue fpSet.setRegion(region) if self.config.nSigmaToGrow > 0: nGrow = int((self.config.nSigmaToGrow * sigma) + 0.5) self.metadata.set("nGrow", nGrow) fpSet = afwDet.FootprintSet(fpSet, nGrow, False) fpSet.setMask(maskedImage.getMask(), maskName) if not self.config.returnOriginalFootprints: setattr(fpSets, polarity, fpSet) fpSets.numPos = len(fpSets.positive.getFootprints() ) if fpSets.positive is not None else 0 fpSets.numNeg = len(fpSets.negative.getFootprints() ) if fpSets.negative is not None else 0 if self.config.thresholdPolarity != "negative": self.log.log( self.log.INFO, "Detected %d positive sources to %g sigma." % (fpSets.numPos, self.config.thresholdValue)) fpSets.background = None if self.config.reEstimateBackground: mi = exposure.getMaskedImage() bkgd = getBackground(mi, self.config.background) if self.config.adjustBackground: self.log.log( self.log.WARN, "Fiddling the background by %g" % self.config.adjustBackground) bkgd += self.config.adjustBackground fpSets.background = bkgd self.log.log( self.log.INFO, "Resubtracting the background after object detection") mi -= bkgd.getImageF() del mi if self.config.thresholdPolarity == "positive": if self.config.reEstimateBackground: mask = maskedImage.getMask() mask &= ~mask.getPlaneBitMask("DETECTED_NEGATIVE") del mask fpSets.negative = None else: self.log.log( self.log.INFO, "Detected %d negative sources to %g %s" % (fpSets.numNeg, self.config.thresholdValue, ("DN" if self.config.thresholdType == "value" else "sigma"))) if display: ds9.mtv(exposure, frame=0, title="detection") if convolvedImage and display and display > 1: ds9.mtv(convolvedImage, frame=1, title="PSF smoothed") if middle and display and display > 1: ds9.mtv(middle, frame=2, title="middle") return fpSets
def matchObjectsToSources(self, refCat, sourceCat, wcs, sourceFluxField, refFluxField, match_tolerance=None): """Match sources to position reference stars. Parameters ---------- refCat : `lsst.afw.table.SimpleCatalog` Reference catalog to match. sourceCat : `lsst.afw.table.SourceCatalog` Catalog of sources found on an exposure. This should already be down-selected to "good"/"usable" sources in the calling Task. wcs : `lsst.afw.geom.SkyWcs` Current WCS of the exposure containing the sources. sourceFluxField : `str` Field of the sourceCat to use for flux refFluxField : `str` Field of the refCat to use for flux match_tolerance : `lsst.meas.astrom.MatchTolerance` Object containing information from previous `lsst.meas.astrom.AstrometryTask` match/fit cycles for use in matching. If `None` is config defaults. Returns ------- matchResult : `lsst.pipe.base.Struct` Result struct with components - ``matches`` : List of matches with distance below the maximum match distance (`list` of `lsst.afw.table.ReferenceMatch`). - ``useableSourceCat`` : Catalog of sources matched and suited for WCS fitting (`lsst.afw.table.SourceCatalog`). - ``match_tolerance`` : MatchTolerance object updated from this match iteration (`lsst.meas.astrom.MatchTolerance`). """ import lsstDebug debug = lsstDebug.Info(__name__) preNumObj = len(refCat) refCat = self.filterStars(refCat) numRefObj = len(refCat) if self.log: self.log.info("filterStars purged %d reference stars, leaving %d stars" % (preNumObj - numRefObj, numRefObj)) if match_tolerance is None: match_tolerance = MatchTolerance() # Make a name alias here for consistency with older code, and to make # it clear that this is a good/usable (cleaned) source catalog. usableSourceCat = sourceCat numUsableSources = len(usableSourceCat) if len(usableSourceCat) == 0: raise pipeBase.TaskError("No sources are usable") minMatchedPairs = min(self.config.minMatchedPairs, int(self.config.minFracMatchedPairs * min([len(refCat), len(usableSourceCat)]))) # match usable (possibly saturated) sources and then purge saturated sources from the match list usableMatches = self._doMatch( refCat=refCat, sourceCat=usableSourceCat, wcs=wcs, refFluxField=refFluxField, numUsableSources=numUsableSources, minMatchedPairs=minMatchedPairs, maxMatchDist=match_tolerance.maxMatchDist, sourceFluxField=sourceFluxField, verbose=debug.verbose, ) # cull non-good sources matches = [] self._getIsGoodKeys(usableSourceCat.schema) for match in usableMatches: if self._isGoodTest(match.second): # Append the isGood match. matches.append(match) self.log.debug("Found %d usable matches, of which %d had good sources", len(usableMatches), len(matches)) if len(matches) == 0: raise RuntimeError("Unable to match sources") self.log.info("Matched %d sources" % len(matches)) if len(matches) < minMatchedPairs: self.log.warn("Number of matches is smaller than request") return pipeBase.Struct( matches=matches, usableSourceCat=usableSourceCat, match_tolerance=match_tolerance, )
def matchObjectsToSources(self, refCat, sourceCat, wcs, refFluxField, match_tolerance=None): """!Match sources to position reference stars @param[in] refCat catalog of reference objects that overlap the exposure; reads fields for: - coord - the specified flux field @param[in] sourceCat catalog of sources found on an exposure; Please check the required fields of your specified source selector that the correct flags are present. @param[in] wcs estimated WCS @param[in] refFluxField field of refCat to use for flux @param[in] match_tolerance is a MatchTolerance class object or None. This this class is used to comunicate state between AstrometryTask and MatcherTask. AstrometryTask will also set the MatchTolerance class variable maxMatchDist based on the scatter AstrometryTask has found after fitting for the wcs. @return an lsst.pipe.base.Struct with fields: - matches a list of matches, each instance of lsst.afw.table.ReferenceMatch - usableSourcCat a catalog of sources potentially usable for matching. - match_tolerance a MatchTolerance object containing the resulting state variables from the match. """ import lsstDebug debug = lsstDebug.Info(__name__) # If we get an empty tolerance struct create the variables we need for # this matcher. if match_tolerance is None: match_tolerance = MatchTolerancePessimistic() # usableSourceCat: sources that are good but may be saturated numSources = len(sourceCat) selectedSources = self.sourceSelector.selectSources(sourceCat) goodSourceCat = selectedSources.sourceCat numUsableSources = len(goodSourceCat) self.log.info("Purged %d sources, leaving %d good sources" % (numSources - numUsableSources, numUsableSources)) if len(goodSourceCat) == 0: raise pipeBase.TaskError("No sources are good") # avoid accidentally using sourceCat; use goodSourceCat from now on del sourceCat minMatchedPairs = min( self.config.minMatchedPairs, int(self.config.minFracMatchedPairs * min([len(refCat), len(goodSourceCat)]))) doMatchReturn = self._doMatch( refCat=refCat, sourceCat=goodSourceCat, wcs=wcs, refFluxField=refFluxField, numUsableSources=numUsableSources, minMatchedPairs=minMatchedPairs, match_tolerance=match_tolerance, sourceFluxField=self.sourceSelector.fluxField, verbose=debug.verbose, ) matches = doMatchReturn.matches match_tolerance = doMatchReturn.match_tolerance if len(matches) == 0: raise RuntimeError("Unable to match sources") self.log.info("Matched %d sources" % len(matches)) if len(matches) < minMatchedPairs: self.log.warn("Number of matches is smaller than request") return pipeBase.Struct( matches=matches, usableSourceCat=goodSourceCat, match_tolerance=match_tolerance, )
def solve(self, exposure, sourceCat): """Load reference objects overlapping an exposure, match to sources and fit a WCS Returns ------- result : `lsst.pipe.base.Struct` Result struct with components: - ``refCat`` : reference object catalog of objects that overlap the exposure (with some margin) (`lsst::afw::table::SimpleCatalog`). - ``matches`` : astrometric matches (`list` of `lsst.afw.table.ReferenceMatch`). - ``scatterOnSky`` : median on-sky separation between reference objects and sources in "matches" (`lsst.geom.Angle`) - ``matchMeta`` : metadata needed to unpersist matches (`lsst.daf.base.PropertyList`) Raises ------ TaskError If the measured mean on-sky distance between the matched source and reference objects is greater than ``self.config.maxMeanDistanceArcsec``. Notes ----- ignores config.forceKnownWcs """ if self.refObjLoader is None: raise RuntimeError( "Running matcher task with no refObjLoader set in __init__ or setRefObjLoader" ) import lsstDebug debug = lsstDebug.Info(__name__) expMd = self._getExposureMetadata(exposure) sourceSelection = self.sourceSelector.run(sourceCat) self.log.info("Purged %d sources, leaving %d good sources", len(sourceCat) - len(sourceSelection.sourceCat), len(sourceSelection.sourceCat)) loadRes = self.refObjLoader.loadPixelBox( bbox=expMd.bbox, wcs=expMd.wcs, filterName=expMd.filterName, epoch=expMd.epoch, ) refSelection = self.referenceSelector.run(loadRes.refCat) matchMeta = self.refObjLoader.getMetadataBox( bbox=expMd.bbox, wcs=expMd.wcs, filterName=expMd.filterName, epoch=expMd.epoch, ) if debug.display: frame = int(debug.frame) displayAstrometry( refCat=refSelection.sourceCat, sourceCat=sourceSelection.sourceCat, exposure=exposure, bbox=expMd.bbox, frame=frame, title="Reference catalog", ) res = None wcs = expMd.wcs match_tolerance = None for i in range(self.config.maxIter): iterNum = i + 1 try: tryRes = self._matchAndFitWcs( refCat=refSelection.sourceCat, sourceCat=sourceCat, goodSourceCat=sourceSelection.sourceCat, refFluxField=loadRes.fluxField, bbox=expMd.bbox, wcs=wcs, exposure=exposure, match_tolerance=match_tolerance, ) except Exception as e: # if we have had a succeessful iteration then use that; otherwise fail if i > 0: self.log.info( "Fit WCS iter %d failed; using previous iteration: %s", iterNum, e) iterNum -= 1 break else: raise match_tolerance = tryRes.match_tolerance tryMatchDist = self._computeMatchStatsOnSky(tryRes.matches) self.log.debug( "Match and fit WCS iteration %d: found %d matches with on-sky distance mean " "= %0.3f +- %0.3f arcsec; max match distance = %0.3f arcsec", iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), tryMatchDist.distStdDev.asArcseconds(), tryMatchDist.maxMatchDist.asArcseconds()) maxMatchDist = tryMatchDist.maxMatchDist res = tryRes wcs = res.wcs if maxMatchDist.asArcseconds( ) < self.config.minMatchDistanceArcSec: self.log.debug( "Max match distance = %0.3f arcsec < %0.3f = config.minMatchDistanceArcSec; " "that's good enough", maxMatchDist.asArcseconds(), self.config.minMatchDistanceArcSec) break match_tolerance.maxMatchDist = maxMatchDist self.log.info( "Matched and fit WCS in %d iterations; " "found %d matches with on-sky distance mean and scatter = %0.3f +- %0.3f arcsec", iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), tryMatchDist.distStdDev.asArcseconds()) if tryMatchDist.distMean.asArcseconds( ) > self.config.maxMeanDistanceArcsec: raise pipeBase.TaskError( "Fatal astrometry failure detected: mean on-sky distance = %0.3f arcsec > %0.3f " "(maxMeanDistanceArcsec)" % (tryMatchDist.distMean.asArcseconds(), self.config.maxMeanDistanceArcsec)) for m in res.matches: if self.usedKey: m.second.set(self.usedKey, True) exposure.setWcs(res.wcs) # Record the scatter in the exposure metadata md = exposure.getMetadata() md['SFM_ASTROM_OFFSET_MEAN'] = tryMatchDist.distMean.asArcseconds() md['SFM_ASTROM_OFFSET_STD'] = tryMatchDist.distStdDev.asArcseconds() return pipeBase.Struct( refCat=refSelection.sourceCat, matches=res.matches, scatterOnSky=res.scatterOnSky, matchMeta=matchMeta, )
def fitBackground(self, maskedImage, nx=0, ny=0, algorithm=None): """!Estimate the background of a masked image @param[in] maskedImage masked image whose background is to be computed @param[in] nx number of x bands; if 0 compute from width and config.binSizeX @param[in] ny number of y bands; if 0 compute from height and config.binSizeY @param[in] algorithm name of interpolation algorithm; if None use self.config.algorithm @return fit background as an lsst.afw.math.Background @throw RuntimeError if lsst.afw.math.makeBackground returns None, which is apparently one way it indicates failure """ binSizeX = self.config.binSize if self.config.binSizeX == 0 else self.config.binSizeX binSizeY = self.config.binSize if self.config.binSizeY == 0 else self.config.binSizeY if not nx: nx = maskedImage.getWidth() // binSizeX + 1 if not ny: ny = maskedImage.getHeight() // binSizeY + 1 unsubFrame = getDebugFrame(self._display, "unsubtracted") if unsubFrame: unsubDisp = afwDisplay.getDisplay(frame=unsubFrame) unsubDisp.mtv(maskedImage, title="unsubtracted") xPosts = numpy.rint( numpy.linspace(0, maskedImage.getWidth() + 1, num=nx, endpoint=True)) yPosts = numpy.rint( numpy.linspace(0, maskedImage.getHeight() + 1, num=ny, endpoint=True)) with unsubDisp.Buffering(): for (xMin, xMax), (yMin, yMax) in itertools.product( zip(xPosts[:-1], xPosts[1:]), zip(yPosts[:-1], yPosts[1:])): unsubDisp.line([(xMin, yMin), (xMin, yMax), (xMax, yMax), (xMax, yMin), (xMin, yMin)]) sctrl = afwMath.StatisticsControl() badMask = maskedImage.mask.getPlaneBitMask( self.config.ignoredPixelMask) sctrl.setAndMask(badMask) sctrl.setNanSafe(self.config.isNanSafe) self.log.debug("Ignoring mask planes: %s" % ", ".join(self.config.ignoredPixelMask)) if (maskedImage.mask.getArray() & badMask).all(): raise pipeBase.TaskError( "All pixels masked. Cannot estimate background") if algorithm is None: algorithm = self.config.algorithm # TODO: DM-22814. This call to a deprecated BackgroundControl constructor # is necessary to support the algorithm parameter; it # should be replaced with # # afwMath.BackgroundControl(nx, ny, sctrl, self.config.statisticsProperty) # # when algorithm has been deprecated and removed. with suppress_deprecations(): bctrl = afwMath.BackgroundControl(algorithm, nx, ny, self.config.undersampleStyle, sctrl, self.config.statisticsProperty) # TODO: The following check should really be done within lsst.afw.math. # With the current code structure, it would need to be accounted for in the doGetImage() # function in BackgroundMI.cc (which currently only checks against the interpolation settings, # which is not appropriate when useApprox=True) # and/or the makeApproximate() function in afw/Approximate.cc. # See ticket DM-2920: "Clean up code in afw for Approximate background # estimation" (which includes a note to remove the following and the # similar checks in pipe_tasks/matchBackgrounds.py once implemented) # # Check that config setting of approxOrder/binSize make sense # (i.e. ngrid (= shortDimension/binSize) > approxOrderX) and perform # appropriate undersampleStlye behavior. if self.config.useApprox: if self.config.approxOrderY not in (self.config.approxOrderX, -1): raise ValueError( "Error: approxOrderY not in (approxOrderX, -1)") order = self.config.approxOrderX minNumberGridPoints = order + 1 if min(nx, ny) <= order: self.log.warn( "Too few points in grid to constrain fit: min(nx, ny) < approxOrder) " "[min(%d, %d) < %d]" % (nx, ny, order)) if self.config.undersampleStyle == "THROW_EXCEPTION": raise ValueError( "Too few points in grid (%d, %d) for order (%d) and binSize (%d, %d)" % (nx, ny, order, binSizeX, binSizeY)) elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER": if order < 1: raise ValueError( "Cannot reduce approxOrder below 0. " "Try using undersampleStyle = \"INCREASE_NXNYSAMPLE\" instead?" ) order = min(nx, ny) - 1 self.log.warn("Reducing approxOrder to %d" % order) elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE": # Reduce bin size to the largest acceptable square bins newBinSize = min( maskedImage.getWidth(), maskedImage.getHeight()) // (minNumberGridPoints - 1) if newBinSize < 1: raise ValueError("Binsize must be greater than 0") newNx = maskedImage.getWidth() // newBinSize + 1 newNy = maskedImage.getHeight() // newBinSize + 1 bctrl.setNxSample(newNx) bctrl.setNySample(newNy) self.log.warn( "Decreasing binSize from (%d, %d) to %d for a grid of (%d, %d)" % (binSizeX, binSizeY, newBinSize, newNx, newNy)) actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order, order, self.config.weighting) bctrl.setApproximateControl(actrl) bg = afwMath.makeBackground(maskedImage, bctrl) if bg is None: raise RuntimeError( "lsst.afw.math.makeBackground failed to fit a background model" ) return bg
def matchObjectsToSources(self, refCat, sourceCat, wcs, refFluxField, match_tolerance=None): """!Match sources to position reference stars @param[in] refCat catalog of reference objects that overlap the exposure; reads fields for: - coord - the specified flux field @param[in] sourceCat catalog of sources found on an exposure; reads fields for: - centroid - centroid flag - edge flag - saturated flag - aperture flux, if found, else PSF flux @param[in] wcs estimated WCS @param[in] refFluxField field of refCat to use for flux @param[in] match_tolerance a MatchTolerance object for specifying tolerances. Must at minimum contain a lsst.afw.geom.Angle called maxMatchDist that communicates state between AstrometryTask and the matcher Task. @return an lsst.pipe.base.Struct with fields: - matches a list of matches, each instance of lsst.afw.table.ReferenceMatch - usableSourcCat a catalog of sources potentially usable for matching. For this fitter usable sources include unresolved sources not too near the edge. It includes saturated sources, even those these are removed from the final match list, because saturated sources may be used to determine the match list. """ import lsstDebug debug = lsstDebug.Info(__name__) preNumObj = len(refCat) refCat = self.filterStars(refCat) numRefObj = len(refCat) if self.log: self.log.info("filterStars purged %d reference stars, leaving %d stars" % (preNumObj - numRefObj, numRefObj)) if match_tolerance is None: match_tolerance = MatchTolerance() # usableSourceCat: sources that are good but may be saturated numSources = len(sourceCat) selectedSources = self.sourceSelector.run(sourceCat) usableSourceCat = selectedSources.sourceCat numUsableSources = len(usableSourceCat) self.log.info("Purged %d unusable sources, leaving %d usable sources" % (numSources - numUsableSources, numUsableSources)) if len(usableSourceCat) == 0: raise pipeBase.TaskError("No sources are usable") del sourceCat # avoid accidentally using sourceCat; use usableSourceCat or goodSourceCat from now on minMatchedPairs = min(self.config.minMatchedPairs, int(self.config.minFracMatchedPairs * min([len(refCat), len(usableSourceCat)]))) # match usable (possibly saturated) sources and then purge saturated sources from the match list usableMatches = self._doMatch( refCat=refCat, sourceCat=usableSourceCat, wcs=wcs, refFluxField=refFluxField, numUsableSources=numUsableSources, minMatchedPairs=minMatchedPairs, maxMatchDist=match_tolerance.maxMatchDist, sourceFluxField=self.sourceSelector.fluxField, verbose=debug.verbose, ) # cull non-good sources matches = [] self._getIsGoodKeys(usableSourceCat.schema) for match in usableMatches: if self._isGoodTest(match.second): # Append the isGood match. matches.append(match) self.log.debug("Found %d usable matches, of which %d had good sources", len(usableMatches), len(matches)) if len(matches) == 0: raise RuntimeError("Unable to match sources") self.log.info("Matched %d sources" % len(matches)) if len(matches) < minMatchedPairs: self.log.warn("Number of matches is smaller than request") return pipeBase.Struct( matches=matches, usableSourceCat=usableSourceCat, match_tolerance=match_tolerance, )
def fitWcs(self, matches, initWcs, bbox=None, refCat=None, sourceCat=None, exposure=None): """!Fit a TAN-SIP WCS from a list of reference object/source matches @param[in,out] matches a list of lsst::afw::table::ReferenceMatch The following fields are read: - match.first (reference object) coord - match.second (source) centroid The following fields are written: - match.first (reference object) centroid, - match.second (source) centroid - match.distance (on sky separation, in radians) @param[in] initWcs initial WCS @param[in] bbox the region over which the WCS will be valid (an lsst:afw::geom::Box2I); if None or an empty box then computed from matches @param[in,out] refCat reference object catalog, or None. If provided then all centroids are updated with the new WCS, otherwise only the centroids for ref objects in matches are updated. Required fields are "centroid_x", "centroid_y", "coord_ra", and "coord_dec". @param[in,out] sourceCat source catalog, or None. If provided then coords are updated with the new WCS; otherwise only the coords for sources in matches are updated. Required fields are "slot_Centroid_x", "slot_Centroid_y", and "coord_ra", and "coord_dec". @param[in] exposure Ignored; present for consistency with FitSipDistortionTask. @return an lsst.pipe.base.Struct with the following fields: - wcs the fit WCS as an lsst.afw.geom.Wcs - scatterOnSky median on-sky separation between reference objects and sources in "matches", as an lsst.afw.geom.Angle """ if bbox is None: bbox = afwGeom.Box2I() import lsstDebug debug = lsstDebug.Info(__name__) wcs = self.initialWcs(matches, initWcs) rejected = np.zeros(len(matches), dtype=bool) for rej in range(self.config.numRejIter): sipObject = self._fitWcs( [mm for i, mm in enumerate(matches) if not rejected[i]], wcs) wcs = sipObject.getNewWcs() rejected = self.rejectMatches(matches, wcs, rejected) if rejected.sum() == len(rejected): raise RuntimeError("All matches rejected in iteration %d" % (rej + 1, )) self.log.debug( "Iteration {0} of astrometry fitting: rejected {1} outliers, " "out of {2} total matches.".format(rej, rejected.sum(), len(rejected))) if debug.plot: print("Plotting fit after rejection iteration %d/%d" % (rej + 1, self.config.numRejIter)) self.plotFit(matches, wcs, rejected) # Final fit after rejection sipObject = self._fitWcs( [mm for i, mm in enumerate(matches) if not rejected[i]], wcs) wcs = sipObject.getNewWcs() if debug.plot: print("Plotting final fit") self.plotFit(matches, wcs, rejected) if refCat is not None: self.log.debug("Updating centroids in refCat") afwTable.updateRefCentroids(wcs, refList=refCat) else: self.log.warn( "Updating reference object centroids in match list; refCat is None" ) afwTable.updateRefCentroids( wcs, refList=[match.first for match in matches]) if sourceCat is not None: self.log.debug("Updating coords in sourceCat") afwTable.updateSourceCoords(wcs, sourceList=sourceCat) else: self.log.warn( "Updating source coords in match list; sourceCat is None") afwTable.updateSourceCoords( wcs, sourceList=[match.second for match in matches]) self.log.debug("Updating distance in match list") setMatchDistance(matches) scatterOnSky = sipObject.getScatterOnSky() if scatterOnSky.asArcseconds() > self.config.maxScatterArcsec: raise pipeBase.TaskError( "Fit failed: median scatter on sky = %0.3f arcsec > %0.3f config.maxScatterArcsec" % (scatterOnSky.asArcseconds(), self.config.maxScatterArcsec)) return pipeBase.Struct( wcs=wcs, scatterOnSky=scatterOnSky, )
def fetchInPatches(self, butler, exposure, tract, patchList, band): """! Get the reference catalogs from a given tract,patchlist This will remove objects where the child is inside the catalog boundary, but the parent is outside the boundary. @param[in] butler A Butler used to get the reference catalogs @param[in] exposure A deepDiff_exposure on which to run the measurements @param[in] tract The tract @param[in] patchList A list of patches that need to be checked . @return Combined SourceCatalog from all the patches """ dataset = f"{self.config.coaddName}Coadd_meas" catalog = None for patch in patchList: dataId = { 'tract': tract.getId(), 'patch': "%d,%d" % patch.getIndex() } dataId['filter'] = band self.log.info("Getting references in %s" % (dataId, )) if not butler.datasetExists(dataset, dataId): if self.config.skipMissing: self.log.info("Could not find %s for dataset %s" % (dataId, dataset)) continue raise pipeBase.TaskError("Reference %s doesn't exist" % (dataId, )) new_catalog = butler.get(dataset, dataId, immediate=True) patchBox = geom.Box2D(patch.getOuterBBox()) tractWcs = tract.getWcs() expBox = geom.Box2D(exposure.getBBox()) expWcs = exposure.getWcs() # only use objects that overlap patch bounding box and overlap exposure validPatch = np.array([ patchBox.contains(tractWcs.skyToPixel(s.getCoord())) for s in new_catalog ]) validExposure = np.array([ expBox.contains(expWcs.skyToPixel(s.getCoord())) for s in new_catalog ]) if validPatch.size == 0 or validExposure.size == 0: self.log.debug("No valid sources %s for dataset %s" % (dataId, dataset)) continue if catalog is None: catalog = new_catalog[validPatch & validExposure] else: catalog.extend(new_catalog[validPatch & validExposure]) if catalog is None: return None # if the parent is not in the catalog remove refCatIdDict = {ref.getId(): ref.getParent() for ref in catalog} refCatIdDict[0] = 0 parentGood = np.array( [refCatIdDict[ref.getId()] in refCatIdDict for ref in catalog]) if np.sum(parentGood == False) > 1: self.log.info("Removing %d/%d objects without parents" % (np.sum(parentGood == False), len(parentGood))) catalog = catalog.copy(deep=True)[parentGood] return catalog