def testMatchXyMatchControl(self): """Test using MatchControl to return all matches, and add a test for closest==False at the same time""" for closest in (True, False): for includeMismatches in (True, False): mc = afwTable.MatchControl() mc.findOnlyClosest = closest mc.includeMismatches = includeMismatches matches = afwTable.matchXy(self.cat1, self.cat2, 0.01, mc) if False: for m in matches: print closest, m.first.getId(), m.second.getId(), m.distance if includeMismatches: catMatches = afwTable.SourceCatalog(self.table) catMismatches = afwTable.SourceCatalog(self.table) for m in matches: if m[1] != None: if not any(x == m[0] for x in catMatches): catMatches.append(m[0]) else: catMismatches.append(m[0]) matches = afwTable.matchXy(catMatches, self.cat2, 0.01, mc) mc.includeMismatches = False noMatches = afwTable.matchXy(catMismatches, self.cat2, 0.01, mc) self.assertEquals(len(noMatches), 0) self.assertEquals(len(matches), self.nUniqueMatch if closest else self.nUniqueMatch + 1) for m in matches: if closest: self.assertEquals(m.first.getId() + self.nobj, m.second.getId()) self.assertEquals(m.distance, 0.0)
def makeMatches(self, refCat, srcCat, nSrc): for i in range(nSrc): refSrc = refCat.addNew() srcSrc = srcCat.addNew() raDeg, decDeg = np.random.randn(2) coord = afwGeom.SpherePoint(raDeg, decDeg, afwGeom.degrees) refSrc.set("g_flux", 10**(-0.4*18)) refSrc.set("r_flux", 10**(-0.4*18)) refSrc.set("resolved", False) refSrc.set("photometric", True) refSrc.setCoord(coord) srcSrc.setCoord(coord) srcSrc.set(srcSrc.getTable().getPsfFluxKey(), 10.) srcSrc.set(srcSrc.getTable().getPsfFluxErrKey(), 1.) for flag in self.sourceSelector.config.badFlags: srcSrc.set(flag, False) mc = afwTable.MatchControl() mc.symmetricMatch = False mat = afwTable.matchRaDec(refCat, srcCat, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(mat), nSrc) return mat
def testNaNPositions(self): ss1 = afwTable.SourceCatalog(self.table) ss2 = afwTable.SourceCatalog(self.table) for ss in (ss1, ss2): ss.addNew().set(afwTable.SourceTable.getCoordKey().getRa(), float('nan') * afwGeom.radians) ss.addNew().set(afwTable.SourceTable.getCoordKey().getDec(), float('nan') * afwGeom.radians) s = ss.addNew() s.set(afwTable.SourceTable.getCoordKey().getRa(), 0.0 * afwGeom.radians) s.set(afwTable.SourceTable.getCoordKey().getDec(), 0.0 * afwGeom.radians) s = ss.addNew() s.set(afwTable.SourceTable.getCoordKey().getRa(), float('nan') * afwGeom.radians) s.set(afwTable.SourceTable.getCoordKey().getDec(), float('nan') * afwGeom.radians) mc = afwTable.MatchControl() mc.findOnlyClosest = False mat = afwTable.matchRaDec(ss1, ss2, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(mat), 1)
def testSelfMatchXy(self): """Test doing a self-matches""" for symmetric in (True, False): mc = afwTable.MatchControl() mc.symmetricMatch = symmetric matches = afwTable.matchXy(self.cat2, 0.01, mc) if False: for m in matches: print m.first.getId(), m.second.getId(), m.distance self.assertEquals(len(matches), 2 if symmetric else 1)
def run(self, butler, coaddSources, ccdInputs, coaddWcs): """ """ if len(self.config.flags) == 0: return flags = self._keys.keys() visitKey = ccdInputs.schema.find("visit").key ccdKey = ccdInputs.schema.find("ccd").key radius = self.config.matchRadius * afwGeom.arcseconds self.log.info("Propagating flags %s from inputs" % (flags, )) counts = dict( (f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags) indices = numpy.array([s.getId() for s in coaddSources ]) # Allowing for non-contiguous data # Accumulate counts of flags being set for ccdRecord in ccdInputs: v = ccdRecord.get(visitKey) c = ccdRecord.get(ccdKey) ccdSources = butler.get("src", run=int(v), ccd=int(c), immediate=True) for sourceRecord in ccdSources: sourceRecord.updateCoord(ccdRecord.getWcs()) for flag in flags: # We assume that the flags will be relatively rare, so it is more efficient to match # against a subset of the input catalog for each flag than it is to match once against # the entire catalog. It would be best to have built a kd-tree on coaddSources and # keep reusing that for the matching, but we don't have a suitable implementation. mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc) for m in matches: index = (numpy.where(indices == m.first.getId()))[0][0] counts[flag][index] += 1 # Apply threshold for f in flags: key = self._keys[f] for s, num in zip(coaddSources, counts[f]): numOverlaps = len( ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True)) s.setFlag(key, bool(num > numOverlaps * self.config.flags[f])) self.log.info("Propagated %d sources with flag %s" % (sum(s.get(key) for s in coaddSources), f))
def testIdentity(self): nobj = 1000 for i in range(nobj): s = self.ss1.addNew() s.setId(i) s.set(afwTable.SourceTable.getCoordKey().getRa(), (10 + 0.001 * i) * afwGeom.degrees) s.set(afwTable.SourceTable.getCoordKey().getDec(), (10 + 0.001 * i) * afwGeom.degrees) s = self.ss2.addNew() s.setId(2 * nobj + i) # Give slight offsets for Coord testing of matches to/from catalog in checkMatchToFromCatalog() # Chosen such that the maximum offset (nobj*1E-7 deg = 0.36 arcsec) is within the maximum # distance (1 arcsec) in afwTable.matchRaDec. s.set(afwTable.SourceTable.getCoordKey().getRa(), (10 + 0.0010001 * i) * afwGeom.degrees) s.set(afwTable.SourceTable.getCoordKey().getDec(), (10 + 0.0010001 * i) * afwGeom.degrees) # Old API (pre DM-855) mat = afwTable.matchRaDec(self.ss1, self.ss2, 1.0 * afwGeom.arcseconds, False) self.assertEqual(len(mat), nobj) # New API mc = afwTable.MatchControl() mc.findOnlyClosest = False mat = afwTable.matchRaDec(self.ss1, self.ss2, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(mat), nobj) cat = afwTable.packMatches(mat) mat2 = afwTable.unpackMatches(cat, self.ss1, self.ss2) for m1, m2, c in zip(mat, mat2, cat): self.assertEqual(m1.first, m2.first) self.assertEqual(m1.second, m2.second) self.assertEqual(m1.distance, m2.distance) self.assertEqual(m1.first.getId(), c["first"]) self.assertEqual(m1.second.getId(), c["second"]) self.assertEqual(m1.distance, c["distance"]) self.checkPickle(mat, checkSlots=False) self.checkPickle(mat2, checkSlots=False) self.checkMatchToFromCatalog(mat, cat) if False: s0 = mat[0][0] s1 = mat[0][1] print(s0.getRa(), s1.getRa(), s0.getId(), s1.getId())
def processCcd(ccdSources, wcsUpdate): for sourceRecord in ccdSources: sourceRecord.updateCoord(wcsUpdate) for flag in flags: # We assume that the flags will be relatively rare, so it is more efficient to match # against a subset of the input catalog for each flag than it is to match once against # the entire catalog. It would be best to have built a kd-tree on coaddSources and # keep reusing that for the matching, but we don't have a suitable implementation. mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc) for m in matches: index = (numpy.where(indices == m.first.getId()))[0][0] counts[flag][index] += 1
def testMismatches(self): """ Chech that matchRaDec works as expected when using the includeMismatches option """ cat1 = afwTable.SourceCatalog(self.table) cat2 = afwTable.SourceCatalog(self.table) nobj = 100 for i in range(nobj): s1 = cat1.addNew() s2 = cat2.addNew() s1.setId(i) s2.setId(i) s1.set(afwTable.SourceTable.getCoordKey().getRa(), (10 + 0.0001 * i) * afwGeom.degrees) s2.set(afwTable.SourceTable.getCoordKey().getRa(), (10.005 + 0.0001 * i) * afwGeom.degrees) s1.set(afwTable.SourceTable.getCoordKey().getDec(), (10 + 0.0001 * i) * afwGeom.degrees) s2.set(afwTable.SourceTable.getCoordKey().getDec(), (10.005 + 0.0001 * i) * afwGeom.degrees) for closest in (True, False): mc = afwTable.MatchControl() mc.findOnlyClosest = closest mc.includeMismatches = False matches = afwTable.matchRaDec(cat1, cat2, 1.0 * afwGeom.arcseconds, mc) mc.includeMismatches = True matchesMismatches = afwTable.matchRaDec(cat1, cat2, 1.0 * afwGeom.arcseconds, mc) catMatches = afwTable.SourceCatalog(self.table) catMismatches = afwTable.SourceCatalog(self.table) for m in matchesMismatches: if m[1] is not None: if not any(x == m[0] for x in catMatches): catMatches.append(m[0]) else: catMismatches.append(m[0]) if closest: self.assertEqual(len(catMatches), len(matches)) matches2 = afwTable.matchRaDec(catMatches, cat2, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(matches), len(matches2)) mc.includeMismatches = False noMatches = afwTable.matchRaDec(catMismatches, cat2, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(noMatches), 0)
def testMatchXyMatchControl(self): """Test using MatchControl to return all matches Also tests closest==False at the same time """ for closest in (True, False): for includeMismatches in (True, False): mc = afwTable.MatchControl() mc.findOnlyClosest = closest mc.includeMismatches = includeMismatches matches = afwTable.matchXy(self.cat1, self.cat2, self.matchRadius, mc) if False: for m in matches: print(closest, m.first.getId(), m.second.getId(), m.distance) if includeMismatches: catMatches = afwTable.SourceCatalog(self.table) catMismatches = afwTable.SourceCatalog(self.table) for m in matches: if m[1] is not None: if not any(x == m[0] for x in catMatches): catMatches.append(m[0]) else: catMismatches.append(m[0]) matches = afwTable.matchXy(catMatches, self.cat2, self.matchRadius, mc) mc.includeMismatches = False noMatches = afwTable.matchXy(catMismatches, self.cat2, self.matchRadius, mc) self.assertEqual(len(noMatches), 0) # If we're not getting only the closest match, then we get an extra match due to the # source we offset by 2 pixels and a bit. Everything else # should match exactly. self.assertEqual( len(matches), self.nUniqueMatch if closest else self.nUniqueMatch + 1) self.assertEqual(sum(1 for m in matches if m.distance == 0.0), self.nUniqueMatch) for m in matches: if closest: self.assertEqual(m.first.getId() + self.nobj, m.second.getId()) else: self.assertLessEqual(m.distance, self.matchRadius)
def testSelfMatchXy(self): """Test doing a self-matches""" for symmetric in (True, False): mc = afwTable.MatchControl() mc.symmetricMatch = symmetric matches = afwTable.matchXy(self.cat2, self.matchRadius, mc) if False: for m in matches: print(m.first.getId(), m.second.getId(), m.distance) # There is only one source that matches another source when we do a self-match: the one # offset by 2 pixels and a bit. # If we're getting symmetric matches, that multiplies the expected number by 2 because it # produces s1,s2 and s2,s1. self.assertEqual(len(matches), 2 if symmetric else 1)
def testIdentity(self): nobj = 1000 for i in range(nobj): s = self.ss1.addNew() s.setId(i) s.set(afwTable.SourceTable.getCoordKey().getRa(), (10 + 0.001 * i) * afwGeom.degrees) s.set(afwTable.SourceTable.getCoordKey().getDec(), (10 + 0.001 * i) * afwGeom.degrees) s = self.ss2.addNew() s.setId(2 * nobj + i) s.set(afwTable.SourceTable.getCoordKey().getRa(), (10 + 0.001 * i) * afwGeom.degrees) s.set(afwTable.SourceTable.getCoordKey().getDec(), (10 + 0.001 * i) * afwGeom.degrees) # Old API (pre DM-855) mat = afwTable.matchRaDec(self.ss1, self.ss2, 1.0 * afwGeom.arcseconds, False) self.assertEqual(len(mat), nobj) # New API mc = afwTable.MatchControl() mc.findOnlyClosest = False mat = afwTable.matchRaDec(self.ss1, self.ss2, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(mat), nobj) cat = afwTable.packMatches(mat) mat2 = afwTable.unpackMatches(cat, self.ss1, self.ss2) for m1, m2, c in zip(mat, mat2, cat): self.assertEqual(m1.first, m2.first) self.assertEqual(m1.second, m2.second) self.assertEqual(m1.distance, m2.distance) self.assertEqual(m1.first.getId(), c["first"]) self.assertEqual(m1.second.getId(), c["second"]) self.assertEqual(m1.distance, c["distance"]) self.checkPickle(mat, checkSlots=False) self.checkPickle(mat2, checkSlots=False) if False: s0 = mat[0][0] s1 = mat[0][1] print s0.getRa(), s1.getRa(), s0.getId(), s1.getId()
def getNoMatchCat(butler, dataType, filters=['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y'], selectSG="/tigress/garmilla/data/cosmos_sg_all.fits", matchRadius=1 * afwGeom.arcseconds, mode='hsc', **kargs): mc = afwTable.MatchControl() mc.includeMismatches = True mc.findOnlyClosest = True sgTable = afwTable.SimpleCatalog.readFits(selectSG) sgTable["coord.ra"][:] = np.radians(sgTable["coord.ra"]) sgTable["coord.dec"][:] = np.radians(sgTable["coord.dec"]) outputCats = [] for f in filters: cat = butler.fetchDataset(dataType, filterSuffix=f, filter=f, **kargs) if mode == 'hsc': schema = cat.getSchema() elif mode == 'hst': schema = sgTable.getSchema() outputCat = afwTable.SimpleCatalog(schema) suffix = _getFilterSuffix(f) if mode == 'hsc': matched = afwTable.matchRaDec(cat, sgTable, matchRadius, mc) elif mode == 'hst': matched = afwTable.matchRaDec(sgTable, cat, matchRadius, mc) for m1, m2, d in matched: if m2 is None: record = outputCat.addNew() record.assign(m1) outputCats.append(outputCat) result = outputCats[0] for i in range(1, len(outputCats)): result.extend(outputCats[i], deep=False) return result
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = int(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius() # sigma of PSF of template image before warping templateSigma = templateExposure.getPsf().computeShape( ).getDeterminantRadius() # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel( ).getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn( "Src product does not exist; running detection, measurement, selection" ) # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma=scienceSigmaPost, doSmooth=not self.doPreConvolve, idFactory=idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len( makeKernelBasisList( self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch( exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(templateSources, selectSources, 1.0 * afwGeom.arcseconds, mc) else: raise RuntimeError( "doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars( exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [ k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize ] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info( "Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId(): x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable( ).getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable( ).getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [ m.first.get(refCoordKey) for m in wcsResults.matches ] residuals = [ m.first.get(refCoordKey).getOffsetFrom( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [ m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] colors = numpy.array([ -2.5 * numpy.log10(srcToMatch[x].get("g")) + 2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys() ]) dlong = numpy.array([ r[0].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) dlat = numpy.array([ r[1].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) idx1 = numpy.where( colors < self.sourceSelector.config.grMin) idx2 = numpy.where( (colors >= self.sourceSelector.config.grMin) & (colors <= self.sourceSelector.config.grMax)) idx3 = numpy.where( colors > self.sourceSelector.config.grMax) rms1Long = IqrToSigma * \ (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25)) rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) - numpy.percentile(dlat[idx1], 25)) rms2Long = IqrToSigma * \ (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25)) rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) - numpy.percentile(dlat[idx2], 25)) rms3Long = IqrToSigma * \ (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25)) rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) - numpy.percentile(dlat[idx3], 25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info( "Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference # Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Computing diffim PSF") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run( exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False. if self.config.doDecorrelation and self.config.doSubtract: decorrResult = self.decorrelate.run(exposure, templateExposure, subtractedExposure, subtractRes.psfMatchingKernel) subtractedExposure = decorrResult.correctedExposure if self.config.doDetection: self.log.info("Running diaSource detection") # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if not self.config.doDipoleFitting: self.measurement.run(diaSources, subtractedExposure) else: if self.config.doSubtract: self.measurement.run(diaSources, subtractedExposure, exposure, subtractRes.matchedExposure) else: self.measurement.run(diaSources, subtractedExposure, exposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs( ).pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn( "Src product does not exist; cannot match with diaSources" ) srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn( "No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info( "Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if sid in srcMatchDict: diaSource.set("srcMatchId", srcMatchDict[sid]) if sid in refMatchDict: diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cand) # Get basis list to build control sample kernels basisList = kernelCandList[0].getKernel( KernelCandidateF.ORIG).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )
def matchCats(cat1, cat2, matchRadius=1 * afwGeom.arcseconds, includeMismatches=False, multiMeas=False, suffix='.2'): """ Match to catalogs and return a catalog with the fields of the two catalogs """ mc = afwTable.MatchControl() mc.includeMismatches = includeMismatches mc.findOnlyClosest = True matched = afwTable.matchRaDec(cat1, cat2, matchRadius, mc) haveCentroid = {} for m1, m2, d in matched: haveCentroid[m1.getId()] = (m1, m2, d) bestMatches = {} if includeMismatches: noMatch = [] for m1, m2, d in matched: if m2 is None: noMatch.append(m1) else: if not multiMeas: id2 = m2.getId() if id2 not in bestMatches: bestMatches[id2] = (m1, m2, d) else: if d < bestMatches[id2][2]: bestMatches[id2] = (m1, m2, d) else: id1 = m1.getId() bestMatches[id1] = (m1, m2, d) if includeMismatches: print "{0} objects from {1} in the first catalog had no match in the second catalog.".format( len(noMatch), len(cat1)) print "{0} objects from the first catalog with a match in the second catalog were not the closest match.".format( len(matched) - len(noMatch) - len(bestMatches)) if includeMismatches and not multiMeas: nMatches = len(cat1) print "I found {0} matches".format(len(bestMatches)) else: nMatches = len(bestMatches) print "I found {0} matches".format(nMatches) schema1 = cat1.getSchema() schema2 = cat2.getSchema() names1 = cat1.schema.getNames() names2 = cat2.schema.getNames() schema = afwTable.SimpleTable.makeMinimalSchema() catKeys = [] cat1Keys = [] cat2Keys = [] for name in names1: cat1Keys.append(schema1.find(name).getKey()) if name not in ['id', 'coord']: catKeys.append(schema.addField(schema1.find(name).getField())) else: catKeys.append(schema.find(name).getKey()) for name in names2: cat2Keys.append(schema2.find(name).getKey()) if name not in schema1.getNames(): catKeys.append(schema.addField(schema2.find(name).getField())) else: catKeys.append( schema.addField( schema2.find(name).getField().copyRenamed(name + suffix))) cat = afwTable.SimpleCatalog(schema) cat.reserve(nMatches) if includeMismatches and not multiMeas: for m1 in cat1: id1 = m1.getId() record = cat.addNew() for i in range(len(cat1Keys)): record.set(catKeys[i], m1.get(cat1Keys[i])) if id1 in haveCentroid: m2 = haveCentroid[id1][1] if m2 is not None: id2 = m2.getId() if id2 in bestMatches: if bestMatches[id2][0] == m1: for i in range(len(cat1Keys), len(catKeys)): record.set(catKeys[i], m2.get(cat2Keys[i - len(cat1Keys)])) else: raise RunTimeError( "If an object in the second catalog has a match it has to be in bestMatches" ) else: for id in bestMatches: m1, m2, d = bestMatches[id] record = cat.addNew() for i in range(len(cat1Keys)): record.set(catKeys[i], m1.get(cat1Keys[i])) for i in range(len(cat1Keys), len(catKeys)): record.set(catKeys[i], m2.get(cat2Keys[i - len(cat1Keys)])) return cat
def run(self, butler, coaddSources, ccdInputs, coaddWcs): """!Propagate flags from individual visit measurements to coadd This requires matching the coadd source catalog to each of the catalogs from the inputs, and thresholding on the number of times a source is flagged on the input catalog. The threshold is made on the relative occurrence of the flag in each source. Flagging a source that is always flagged in inputs corresponds to a threshold of 1, while flagging a source that is flagged in any of the input corresponds to a threshold of 0. But neither of these extrema are really useful in practise. Setting the threshold too high means that sources that are not consistently flagged (e.g., due to chip gaps) will not have the flag propagated. Setting that threshold too low means that random sources which are falsely flagged in the inputs will start to dominate. If in doubt, we suggest making this threshold relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more confidence in the quality of the flagging, the lower the threshold can be. The relative occurrence accounts for the edge of the field-of-view of the camera, but does not include chip gaps, bad or saturated pixels, etc. @param[in] butler Data butler, for retrieving the input source catalogs @param[in,out] coaddSources Source catalog from the coadd @param[in] ccdInputs Table of CCDs that contribute to the coadd @param[in] coaddWcs Wcs for coadd """ if len(self.config.flags) == 0: return flags = self._keys.keys() visitKey = ccdInputs.schema.find("visit").key ccdKey = ccdInputs.schema.find("ccd").key radius = self.config.matchRadius * afwGeom.arcseconds self.log.info("Propagating flags %s from inputs" % (flags, )) counts = dict( (f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags) indices = numpy.array([s.getId() for s in coaddSources ]) # Allowing for non-contiguous data # Accumulate counts of flags being set for ccdRecord in ccdInputs: v = ccdRecord.get(visitKey) c = ccdRecord.get(ccdKey) ccdSources = butler.get("src", visit=int(v), ccd=int(c), immediate=True) for sourceRecord in ccdSources: sourceRecord.updateCoord(ccdRecord.getWcs()) for flag in flags: # We assume that the flags will be relatively rare, so it is more efficient to match # against a subset of the input catalog for each flag than it is to match once against # the entire catalog. It would be best to have built a kd-tree on coaddSources and # keep reusing that for the matching, but we don't have a suitable implementation. mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc) for m in matches: index = (numpy.where(indices == m.first.getId()))[0][0] counts[flag][index] += 1 # Apply threshold for f in flags: key = self._keys[f] for s, num in zip(coaddSources, counts[f]): numOverlaps = len( ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True)) s.setFlag(key, bool(num > numOverlaps * self.config.flags[f])) self.log.info("Propagated %d sources with flag %s" % (sum(s.get(key) for s in coaddSources), f))
def copyIcSourceFields(self, icSourceCat, sourceCat): """!Match sources in icSourceCat and sourceCat and copy the specified fields @param[in] icSourceCat catalog from which to copy fields @param[in,out] sourceCat catalog to which to copy fields The fields copied are those specified by `config.icSourceFieldsToCopy` that actually exist in the schema. This was set up by the constructor using self.schemaMapper. """ if self.schemaMapper is None: raise RuntimeError("To copy icSource fields you must specify " "icSourceSchema nd icSourceKeys when " "constructing this task") if icSourceCat is None or sourceCat is None: raise RuntimeError("icSourceCat and sourceCat must both be " "specified") if len(self.config.icSourceFieldsToCopy) == 0: self.log.warn("copyIcSourceFields doing nothing because " "icSourceFieldsToCopy is empty") return mc = afwTable.MatchControl() mc.findOnlyClosest = False # return all matched objects matches = afwTable.matchXy(icSourceCat, sourceCat, self.config.matchRadiusPix, mc) if self.config.doDeblend: deblendKey = sourceCat.schema["deblend_nChild"].asKey() # if deblended, keep children matches = [m for m in matches if m[1].get(deblendKey) == 0] # Because we had to allow multiple matches to handle parents, we now # need to prune to the best matches # closest matches as a dict of icSourceCat source ID: # (icSourceCat source, sourceCat source, distance in pixels) bestMatches = {} for m0, m1, d in matches: id0 = m0.getId() match = bestMatches.get(id0) if match is None or d <= match[2]: bestMatches[id0] = (m0, m1, d) matches = list(bestMatches.values()) # Check that no sourceCat sources are listed twice (we already know # that each match has a unique icSourceCat source ID, due to using # that ID as the key in bestMatches) numMatches = len(matches) numUniqueSources = len(set(m[1].getId() for m in matches)) if numUniqueSources != numMatches: self.log.warn("{} icSourceCat sources matched only {} sourceCat " "sources".format(numMatches, numUniqueSources)) self.log.info("Copying flags from icSourceCat to sourceCat for " "%s sources" % (numMatches,)) # For each match: set the calibSourceKey flag and copy the desired # fields for icSrc, src, d in matches: src.setFlag(self.calibSourceKey, True) # src.assign copies the footprint from icSrc, which we don't want # (DM-407) # so set icSrc's footprint to src's footprint before src.assign, # then restore it icSrcFootprint = icSrc.getFootprint() try: icSrc.setFootprint(src.getFootprint()) src.assign(icSrc, self.schemaMapper) finally: icSrc.setFootprint(icSrcFootprint)
def buildXY(hscCat, sgTable, matchRadius=1 * afwGeom.arcseconds, includeMismatches=True, multiMeas=False): mc = afwTable.MatchControl() mc.includeMismatches = includeMismatches mc.findOnlyClosest = True print "Matching with HST catalog" matchedSG = afwTable.matchRaDec(hscCat, sgTable, matchRadius, mc) print "Found {0} matches with HST objects".format(len(matchedSG)) # Build truth table stellar = {} classKey = sgTable.getSchema().find('mu.class').key magAutoKey = sgTable.getSchema().find('mag.auto').key noMatch = [] for m1, m2, d in matchedSG: if m2 is None: noMatch.append(m1.getId()) else: if not multiMeas: id = m2.getId() isStar = (m2.get(classKey) == 2) magAuto = m2.get(magAutoKey) if id not in stellar: stellar[id] = [isStar, magAuto, d, m1] else: if d < stellar[id][2]: stellar[id] = [isStar, magAuto, d, m1] # Only keep closest for now else: id = m1.getId() isStar = (m2.get(classKey) == 2) magAuto = m2.get(magAutoKey) stellar[id] = [isStar, magAuto, d, m1] if includeMismatches: print "{0} objects from {1} in the HSC catalog had no match in the HST catalog.".format( len(noMatch), len(hscCat)) print "{0} objects from the HSC catalog with a match in the HST catalog were not the closest match.".format( len(matchedSG) - len(noMatch) - len(stellar)) print "Of which I picked {0}".format(len(stellar)) scm = createSchemaMapper(hscCat, withStellar=True) schema = scm.getOutputSchema() cat = afwTable.SourceCatalog(schema) cat.reserve(len(stellar)) stellarKey = schema.find('stellar').key magAutoKey = schema.find('mag.auto').key for id in stellar: isStar, magAuto, d, m2 = stellar[id] record = cat.addNew() record.assign(m2, scm) record.set(stellarKey, isStar) record.set(magAutoKey, magAuto) if includeMismatches: return cat, noMatch return cat
def strictMatch(cat1, cat2, matchRadius=1 * afwGeom.arcseconds, includeMismatches=True, multiMeas=False): """ Match two catalogs using a one to one relation where each match is the closest object """ mc = afwTable.MatchControl() mc.includeMismatches = includeMismatches mc.findOnlyClosest = True #matched = afwTable.matchRaDec(cat1, cat2, matchRadius, True) matched = afwTable.matchRaDec(cat1, cat2, matchRadius, mc) bestMatches = {} noMatch = [] for m1, m2, d in matched: if m2 is None: noMatch.append(m1) else: if not multiMeas: id = m2.getId() if id not in bestMatches: bestMatches[id] = (m1, m2, d) else: if d < bestMatches[id][2]: bestMatches[id] = (m1, m2, d) else: id = m1.getId() bestMatches[id] = (m1, m2, d) if includeMismatches: print "{0} objects from {1} in the first catalog had no match in the second catalog.".format( len(noMatch), len(cat1)) print "{0} objects from the first catalog with a match in the second catalog were not the closest match.".format( len(matched) - len(noMatch) - len(bestMatches)) scm = createSchemaMapper(cat1, cat2) schema = scm.getOutputSchema() cat = afwTable.SimpleCatalog(schema) cat.reserve(len(bestMatches)) cat2Fields = [] cat2Keys = [] catKeys = [] schema2 = cat2.getSchema() suffixes = getCatSuffixes(cat2) for suffix in suffixes: cat2Fields.extend(schema2.extract("*" + suffix).keys()) for f in cat2Fields: cat2Keys.append(schema2.find(f).key) catKeys.append(schema.find(f).key) for id in bestMatches: m1, m2, d = bestMatches[id] record = cat.addNew() record.assign(m1, scm) for i in range(len(cat2Keys)): record.set(catKeys[i], m2.get(cat2Keys[i])) return cat
def testPhotometricCalib(self): """Test matching the CFHT catalogue (as generated using LSST code) to the SDSS catalogue """ band = 2 # SDSS r # Read SDSS catalogue with open(os.path.join(afwdataDir, "CFHT", "D2", "sdss.dat"), "r") as ifd: sdss = afwTable.SourceCatalog(self.table) sdssSecondary = afwTable.SourceCatalog(self.table) PRIMARY, SECONDARY = 1, 2 # values of mode id = 0 for line in ifd.readlines(): if re.search(r"^\s*#", line): continue fields = line.split() objId = int(fields[0]) fields[1] mode = int(fields[2]) ra, dec = [float(f) for f in fields[3:5]] psfMags = [float(f) for f in fields[5:]] if mode == PRIMARY: s = sdss.addNew() elif SECONDARY: s = sdssSecondary.addNew() s.setId(objId) s.setRa(ra * afwGeom.degrees) s.setDec(dec * afwGeom.degrees) s.set(self.table.getPsfFluxKey(), psfMags[band]) del ifd # Read catalalogue built from the template image # Read SDSS catalogue with open(os.path.join(afwdataDir, "CFHT", "D2", "template.dat"), "r") as ifd: template = afwTable.SourceCatalog(self.table) id = 0 for line in ifd.readlines(): if re.search(r"^\s*#", line): continue fields = line.split() id, flags = [int(f) for f in fields[0:2]] ra, dec = [float(f) for f in fields[2:4]] flux = [float(f) for f in fields[4:]] if flags & 0x1: # EDGE continue s = template.addNew() s.setId(id) id += 1 s.set(afwTable.SourceTable.getCoordKey().getRa(), ra * afwGeom.degrees) s.set(afwTable.SourceTable.getCoordKey().getDec(), dec * afwGeom.degrees) s.set(self.table.getPsfFluxKey(), flux[0]) del ifd # Actually do the match mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(sdss, template, 1.0 * afwGeom.arcseconds, mc) self.assertEqual(len(matches), 901) if False: for mat in matches: s0 = mat[0] s1 = mat[1] d = mat[2] print(s0.getRa(), s0.getDec(), s1.getRa(), s1.getDec(), s0.getPsfFlux(), s1.getPsfFlux()) # Actually do the match for s in sdssSecondary: sdss.append(s) mc = afwTable.MatchControl() mc.symmetricMatch = False matches = afwTable.matchRaDec(sdss, 1.0 * afwGeom.arcseconds, mc) nmiss = 1 # one object doesn't match self.assertEqual(len(matches), len(sdssSecondary) - nmiss) # Find the one that didn't match if False: matchIds = set() for s0, s1, d in matches: matchIds.add(s0.getId()) matchIds.add(s1.getId()) for s in sdssSecondary: if s.getId() not in matchIds: print("RHL", s.getId()) matches = afwTable.matchRaDec(sdss, 1.0 * afwGeom.arcseconds) self.assertEqual(len(matches), 2 * (len(sdssSecondary) - nmiss)) if False: for mat in matches: s0 = mat[0] s1 = mat[1] mat[2] print(s0.getId(), s1.getId(), s0.getRa(), s0.getDec(), end=' ') print(s1.getRa(), s1.getDec(), s0.getPsfFlux(), s1.getPsfFlux())