def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3): sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True) sIm *= scaling kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution( ipDiffim.KernelCandidateF.RECENT), kSum=scaling) sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True) sIm += bg kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution( ipDiffim.KernelCandidateF.RECENT), bg=bg)
def testGaussianWithNoise(self): # Convolve a real image with a gaussian and try and recover # it. Add noise and perform the same test. gsize = self.policy.getInt("kernelSize") gaussFunction = afwMath.GaussianFunction2D(2, 3) gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction) kImageIn = afwImage.ImageD(afwGeom.Extent2I(gsize, gsize)) kSumIn = gaussKernel.computeImage(kImageIn, False) imX, imY = self.templateExposure2.getMaskedImage().getDimensions() smi = afwImage.MaskedImageF(afwGeom.Extent2I(imX, imY)) afwMath.convolve(smi, self.templateExposure2.getMaskedImage(), gaussKernel, False) bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL)) tmi2 = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), bbox, origin=afwImage.LOCAL) smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL) kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) kImageOut = kc.getImage() soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) self.assertAlmostEqual(soln.getKsum(), kSumIn) # 8.7499380640430563e-06 != 0.0 within 7 places self.assertAlmostEqual(soln.getBackground(), 0.0, 4) for j in range(kImageOut.getHeight()): for i in range(kImageOut.getWidth()): # in the outskirts of the kernel, the ratio can get screwed because of low S/N # e.g. 7.45817359824e-09 vs. 1.18062529402e-08 # in the guts of the kernel it should look closer if kImageIn[i, j, afwImage.LOCAL] > 1e-4: # sigh, too bad this sort of thing fails.. # 0.99941584433815966 != 1.0 within 3 places self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL]/kImageIn[i, j, afwImage.LOCAL], 1.0, 2) # now repeat with noise added; decrease precision of comparison self.addNoise(smi2) kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) kImageOut = kc.getImage() soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) self.assertAlmostEqual(soln.getKsum(), kSumIn, 3) if not (self.policy.get("fitForBackground")): self.assertEqual(soln.getBackground(), 0.0) for j in range(kImageOut.getHeight()): for i in range(kImageOut.getWidth()): if kImageIn[i, j, afwImage.LOCAL] > 1e-2: self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL], kImageIn[i, j, afwImage.LOCAL], 2)
def testGaussianWithNoise(self): # Convolve a real image with a gaussian and try and recover # it. Add noise and perform the same test. gsize = self.ps["kernelSize"] gaussFunction = afwMath.GaussianFunction2D(2, 3) gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction) kImageIn = afwImage.ImageD(geom.Extent2I(gsize, gsize)) kSumIn = gaussKernel.computeImage(kImageIn, False) imX, imY = self.templateExposure2.getMaskedImage().getDimensions() smi = afwImage.MaskedImageF(geom.Extent2I(imX, imY)) afwMath.convolve(smi, self.templateExposure2.getMaskedImage(), gaussKernel, False) bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL)) tmi2 = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), bbox, origin=afwImage.LOCAL) smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL) kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) kImageOut = kc.getImage() soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) self.assertAlmostEqual(soln.getKsum(), kSumIn) # 8.7499380640430563e-06 != 0.0 within 7 places self.assertAlmostEqual(soln.getBackground(), 0.0, 4) for j in range(kImageOut.getHeight()): for i in range(kImageOut.getWidth()): # in the outskirts of the kernel, the ratio can get screwed because of low S/N # e.g. 7.45817359824e-09 vs. 1.18062529402e-08 # in the guts of the kernel it should look closer if kImageIn[i, j, afwImage.LOCAL] > 1e-4: # sigh, too bad this sort of thing fails.. # 0.99941584433815966 != 1.0 within 3 places self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL]/kImageIn[i, j, afwImage.LOCAL], 1.0, 2) # now repeat with noise added; decrease precision of comparison self.addNoise(smi2) kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) kImageOut = kc.getImage() soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) self.assertAlmostEqual(soln.getKsum(), kSumIn, 3) if not self.ps.get("fitForBackground"): self.assertEqual(soln.getBackground(), 0.0) for j in range(kImageOut.getHeight()): for i in range(kImageOut.getWidth()): if kImageIn[i, j, afwImage.LOCAL] > 1e-2: self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL], kImageIn[i, j, afwImage.LOCAL], 2)
def testDeltaFunctionScaled(self, scaling = 2.7, bg = 11.3): if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True) sIm *= scaling kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT), kSum = scaling) sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True) sIm += bg kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT), bg = bg)
def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3): if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True) sIm *= scaling kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution( ipDiffim.KernelCandidateF.RECENT), kSum=scaling) sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True) sIm += bg kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution( ipDiffim.KernelCandidateF.RECENT), bg=bg)
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "DF" self.subconfig = self.config.kernel.active self.policy = pexConfig.makePolicy(self.subconfig) self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
def testDeltaFunction(self): if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return # Match an image to itself, with delta-function basis set # No regularization kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), self.templateExposure2.getMaskedImage(), self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) # These should work for kType in (ipDiffim.KernelCandidateF.ORIG, ipDiffim.KernelCandidateF.RECENT): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception, e: print kMethod, e self.fail() else: pass
def testDeltaFunction(self): if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return # Match an image to itself, with delta-function basis set # No regularization kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), self.templateExposure2.getMaskedImage(), self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) # These should work for kType in (ipDiffim.KernelCandidateF.ORIG, ipDiffim.KernelCandidateF.RECENT): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception, e: print kMethod, e self.fail() else: pass
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "DF" self.subconfig = self.config.kernel.active self.ps = pexConfig.makePropertySet(self.subconfig) self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
def testSubtractMaskedImages(self): # Lets do some additional testing here to make sure we recover # the known spatial model. No background, just the faked # alard-lupton basis set. The rest of matchMaskedImages() and # subtractMaskedImages() functionality is tested by the # Exposure-based methods. fakeCoeffs = diffimTools.fakeCoeffs() # Quick note; you shouldn't change confake here, since the # candidates in the KernelCellSet are initialized in # makeFakeKernelSet tMi, sMi, sK, kcs, confake = diffimTools.makeFakeKernelSet(bgValue = 0.0, addNoise = False) svar = sMi.getVariance() svar.set(1.0) tvar = tMi.getVariance() tvar.set(1.0) basisList = ipDiffim.makeKernelBasisList(confake.kernel.active) psfMatchAL = ipDiffim.ImagePsfMatchTask(config=confake) spatialSolution, psfMatchingKernel, backgroundModel = psfMatchAL._solve(kcs, basisList) fitCoeffs = psfMatchingKernel.getSpatialParameters() for b in range(len(fakeCoeffs)): for s in range(len(fakeCoeffs[b])): if fakeCoeffs[b][s] == 0.0: self.assertAlmostEqual(fitCoeffs[b][s], 0.0) else: # OUTSTANDING ISSUE - WHY IS THIS ONE TERM OFF!?!? if b != 4 and s != 0: self.assertAlmostEqual(fitCoeffs[b][s]/fakeCoeffs[b][s], 1.0, 1)
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "DF" self.subconfig = self.config.kernel.active self.kList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy = pexConfig.makePolicy(self.subconfig) self.policy.set("useRegularization", False)
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "AL" self.subconfig = self.config.kernel.active self.policy = pexConfig.makePolicy(self.subconfig) self.kList = ipDiffim.makeKernelBasisList(self.subconfig) self.ksize = self.policy.get('kernelSize')
def testDeltaFunction(self): # Match an image to itself, with delta-function basis set # No regularization kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), self.templateExposure2.getMaskedImage(), self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) # These should work for kType in (ipDiffim.KernelCandidateF.ORIG, ipDiffim.KernelCandidateF.RECENT): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception as e: print(kMethod, e) self.fail() else: pass try: kc.getImage() except Exception as e: print(kMethod, e) self.fail() else: pass # None of these should work for kType in (ipDiffim.KernelCandidateF.PCA,): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getImage, kc.getDifferenceImage): try: kMethod(kType) except Exception: pass else: print(kMethod) self.fail() self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT))
def testGaussian(self, imsize=50): # Convolve a delta function with a known gaussian; try to # recover using delta-function basis gsize = self.ps["kernelSize"] tsize = imsize + gsize gaussFunction = afwMath.GaussianFunction2D(2, 3) gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction) kImageIn = afwImage.ImageD(geom.Extent2I(gsize, gsize)) gaussKernel.computeImage(kImageIn, False) # template image with a single hot pixel in the exact center tmi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize)) tmi.set(0, 0x0, 1e-4) cpix = tsize // 2 tmi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 1) # science image smi = afwImage.MaskedImageF(tmi.getDimensions()) convolutionControl = afwMath.ConvolutionControl() convolutionControl.setDoNormalize(False) afwMath.convolve(smi, tmi, gaussKernel, convolutionControl) # get the actual kernel sum (since the image is not infinite) gscaling = afwMath.makeStatistics(smi, afwMath.SUM).getValue(afwMath.SUM) # grab only the non-masked subregion bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL)) tmi2 = afwImage.MaskedImageF(tmi, bbox, origin=afwImage.LOCAL) smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL) # make sure its a valid subregion! for j in range(tmi2.getHeight()): for i in range(tmi2.getWidth()): self.assertEqual(tmi2.mask[i, j, afwImage.LOCAL], 0) self.assertEqual(smi2.mask[i, j, afwImage.LOCAL], 0) kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi2, smi2, self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) kImageOut = kc.getImage() soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) self.assertAlmostEqual(soln.getKsum(), gscaling) self.assertAlmostEqual(soln.getBackground(), 0.0) for j in range(kImageOut.getHeight()): for i in range(kImageOut.getWidth()): self.assertAlmostEqual( kImageOut[i, j, afwImage.LOCAL] / kImageIn[i, j, afwImage.LOCAL], 1.0, 5)
def testDeltaFunction(self): # Match an image to itself, with delta-function basis set # No regularization kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), self.templateExposure2.getMaskedImage(), self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) # These should work for kType in (ipDiffim.KernelCandidateF.ORIG, ipDiffim.KernelCandidateF.RECENT): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception as e: print(kMethod, e) self.fail() else: pass try: kc.getImage() except Exception as e: print(kMethod, e) self.fail() else: pass # None of these should work for kType in (ipDiffim.KernelCandidateF.PCA,): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getImage, kc.getDifferenceImage): try: kMethod(kType) except Exception: pass else: print(kMethod) self.fail() self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT))
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "DF" self.subconfig = self.config.kernel.active self.policy = pexConfig.makePolicy(self.subconfig) self.policy.set("useRegularization", False) self.policy.set("checkConditionNumber", False) # I am making shady kernels by hand self.policy.set("useCoreStats", False) # I am making off-center resids self.kList = ipDiffim.makeKernelBasisList(self.subconfig) self.size = 51
def testGaussian(self, imsize = 50): # Convolve a delta function with a known gaussian; try to # recover using delta-function basis gsize = self.policy.getInt("kernelSize") tsize = imsize + gsize gaussFunction = afwMath.GaussianFunction2D(2, 3) gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction) kImageIn = afwImage.ImageD(afwGeom.Extent2I(gsize, gsize)) gaussKernel.computeImage(kImageIn, False) # template image with a single hot pixel in the exact center tmi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize)) tmi.set(0, 0x0, 1e-4) cpix = tsize // 2 tmi.set(cpix, cpix, (1, 0x0, 1)) # science image smi = afwImage.MaskedImageF(tmi.getDimensions()) afwMath.convolve(smi, tmi, gaussKernel, False) # get the actual kernel sum (since the image is not infinite) gscaling = afwMath.makeStatistics(smi, afwMath.SUM).getValue(afwMath.SUM) # grab only the non-masked subregion bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL)) tmi2 = afwImage.MaskedImageF(tmi, bbox, afwImage.LOCAL) smi2 = afwImage.MaskedImageF(smi, bbox, afwImage.LOCAL) # make sure its a valid subregion! for j in range(tmi2.getHeight()): for i in range(tmi2.getWidth()): self.assertEqual(tmi2.getMask().get(i, j), 0) self.assertEqual(smi2.getMask().get(i, j), 0) kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi2, smi2, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) kImageOut = kc.getImage() soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT) self.assertAlmostEqual(soln.getKsum(), gscaling) self.assertAlmostEqual(soln.getBackground(), 0.0) for j in range(kImageOut.getHeight()): for i in range(kImageOut.getWidth()): self.assertAlmostEqual(kImageOut.get(i, j)/kImageIn.get(i, j), 1.0, 5)
def testSourceStats(self): source = self.ss.addNew() source.setId(1) source.set(self.table.getCentroidKey().getX(), 276) source.set(self.table.getCentroidKey().getY(), 717) source.set("slot_PsfFlux_instFlux", 1.) kc = ipDiffim.KernelCandidateF(source, self.templateExposure2.getMaskedImage(), self.scienceImage2.getMaskedImage(), self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True)
def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3): sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True) sIm *= scaling kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT), kSum=scaling) sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True) sIm += bg kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT), bg=bg)
def testSourceStats(self): source = self.ss.addNew() source.setId(1) source.set(self.table.getCentroidSlot().getMeasKey().getX(), 276) source.set(self.table.getCentroidSlot().getMeasKey().getY(), 717) source.set("slot_PsfFlux_instFlux", 1.) kc = ipDiffim.KernelCandidateF(source, self.templateExposure2.getMaskedImage(), self.scienceImage2.getMaskedImage(), self.ps) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True)
def runAlSpatialModel(self, sko, bgo): basisList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy.set('spatialKernelOrder', sko) self.policy.set('spatialBgOrder', bgo) self.policy.set('fitForBackground', True) bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(self.size * 10, self.size * 10)) bsikv = ipDiffim.BuildSingleKernelVisitorF(basisList, self.policy) bspkv = ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) for x in range(1, self.size, 10): for y in range(1, self.size, 10): cand = self.makeCandidate(1.0, x, y) bsikv.processCandidate(cand) bspkv.processCandidate(cand) bspkv.solveLinearEquation() sk, sb = bspkv.getSolutionPair() # Kernel if sko == 0: # Specialization for speedup spatialKernelSolution = sk.getKernelParameters() # One term for each basis function self.assertEqual(len(spatialKernelSolution), len(basisList)) else: spatialKernelSolution = sk.getSpatialParameters() nSpatialTerms = int(0.5 * (sko + 1) * (sko + 2)) # One model for each basis function self.assertEqual(len(spatialKernelSolution), len(basisList)) # First basis has no spatial variation for i in range(1, nSpatialTerms): self.assertEqual(spatialKernelSolution[0][i], 0.) # All bases have correct number of terms for i in range(len(spatialKernelSolution)): self.assertEqual(len(spatialKernelSolution[i]), nSpatialTerms) # Background spatialBgSolution = sb.getParameters() nBgTerms = int(0.5 * (bgo + 1) * (bgo + 2)) self.assertEqual(len(spatialBgSolution), nBgTerms)
def testModelType(self): bbox = geom.Box2I(geom.Point2I(10, 10), geom.Extent2I(10, 10)) basisList = ipDiffim.makeKernelBasisList(self.subconfig) self.ps["spatialModelType"] = "polynomial" ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.ps) self.ps["spatialModelType"] = "chebyshev1" ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.ps) try: self.ps["spatialModelType"] = "foo" ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.ps) except Exception: pass else: self.fail()
def testModelType(self): bbox = afwGeom.Box2I(afwGeom.Point2I(10, 10), afwGeom.Extent2I(10, 10)) basisList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy.set("spatialModelType", "polynomial") ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) self.policy.set("spatialModelType", "chebyshev1") ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) try: self.policy.set("spatialModelType", "foo") ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) except Exception: pass else: self.fail()
def runAlSpatialModel(self, sko, bgo): basisList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy.set('spatialKernelOrder', sko) self.policy.set('spatialBgOrder', bgo) self.policy.set('fitForBackground', True) bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(self.size*10, self.size*10)) bsikv = ipDiffim.BuildSingleKernelVisitorF(basisList, self.policy) bspkv = ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) for x in range(1, self.size, 10): for y in range(1, self.size, 10): cand = self.makeCandidate(1.0, x, y) bsikv.processCandidate(cand) bspkv.processCandidate(cand) bspkv.solveLinearEquation() sk, sb = bspkv.getSolutionPair() # Kernel if sko == 0: # Specialization for speedup spatialKernelSolution = sk.getKernelParameters() # One term for each basis function self.assertEqual(len(spatialKernelSolution), len(basisList)) else: spatialKernelSolution = sk.getSpatialParameters() nSpatialTerms = int(0.5 * (sko + 1) * (sko + 2)) # One model for each basis function self.assertEqual(len(spatialKernelSolution), len(basisList)) # First basis has no spatial variation for i in range(1, nSpatialTerms): self.assertEqual(spatialKernelSolution[0][i], 0.) # All bases have correct number of terms for i in range(len(spatialKernelSolution)): self.assertEqual(len(spatialKernelSolution[i]), nSpatialTerms) # Background spatialBgSolution = sb.getParameters() nBgTerms = int(0.5 * (bgo + 1) * (bgo + 2)) self.assertEqual(len(spatialBgSolution), nBgTerms)
def testModelType(self): bbox = afwGeom.Box2I(afwGeom.Point2I(10, 10), afwGeom.Extent2I(10, 10)) basisList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy.set("spatialModelType", "polynomial") ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) self.policy.set("spatialModelType", "chebyshev1") ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) try: self.policy.set("spatialModelType", "foo") ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy) except Exception: pass else: self.fail()
def testSourceStats(self): # Original and uninitialized if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return source = self.ss.addNew() source.setId(1) source.set(self.table.getCentroidKey().getX(), 276) source.set(self.table.getCentroidKey().getY(), 717) source.set(self.table.getPsfFluxKey(), 1.) kc = ipDiffim.KernelCandidateF(source, self.templateExposure2.getMaskedImage(), self.scienceImage2.getMaskedImage(), self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True)
def testSourceStats(self): # Original and uninitialized if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return source = self.ss.addNew() source.setId(1) source.set(self.table.getCentroidKey().getX(), 276) source.set(self.table.getCentroidKey().getY(), 717) source.set(self.table.getPsfFluxKey(), 1.) kc = ipDiffim.KernelCandidateF(source, self.templateExposure2.getMaskedImage(), self.scienceImage2.getMaskedImage(), self.policy) kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True)
def testZeroVariance(self, imsize=50): gsize = self.ps["kernelSize"] tsize = imsize + gsize tmi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize)) tmi.set(0, 0x0, 1.0) cpix = tsize // 2 tmi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 0.0) smi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize)) smi.set(0, 0x0, 1.0) smi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 0.0) kList = ipDiffim.makeKernelBasisList(self.subconfig) self.ps["constantVarianceWeighting"] = False kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.ps) try: kc.build(kList) except Exception: pass else: self.fail()
def testZeroVariance(self, imsize=50): gsize = self.policy.getInt("kernelSize") tsize = imsize + gsize tmi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize)) tmi.set(0, 0x0, 1.0) cpix = tsize // 2 tmi.set(cpix, cpix, (1, 0x0, 0.0)) smi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize)) smi.set(0, 0x0, 1.0) smi.set(cpix, cpix, (1, 0x0, 0.0)) kList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy.set("constantVarianceWeighting", False) kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.policy) try: kc.build(kList) except Exception: pass else: self.fail()
def testZeroVariance(self, imsize = 50): gsize = self.policy.getInt("kernelSize") tsize = imsize + gsize tmi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize)) tmi.set(0, 0x0, 1.0) cpix = tsize // 2 tmi.set(cpix, cpix, (1, 0x0, 0.0)) smi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize)) smi.set(0, 0x0, 1.0) smi.set(cpix, cpix, (1, 0x0, 0.0)) kList = ipDiffim.makeKernelBasisList(self.subconfig) self.policy.set("constantVarianceWeighting", False) kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.policy) try: kc.build(kList) except Exception: pass else: self.fail()
def testSubtractMaskedImages(self): # Lets do some additional testing here to make sure we recover # the known spatial model. No background, just the faked # alard-lupton basis set. The rest of matchMaskedImages() and # subtractMaskedImages() functionality is tested by the # Exposure-based methods. fakeCoeffs = diffimTools.fakeCoeffs() # Quick note; you shouldn't change confake here, since the # candidates in the KernelCellSet are initialized in # makeFakeKernelSet tMi, sMi, sK, kcs, confake = diffimTools.makeFakeKernelSet( bgValue=0.0, addNoise=False) svar = sMi.getVariance() svar.set(1.0) tvar = tMi.getVariance() tvar.set(1.0) basisList = ipDiffim.makeKernelBasisList(confake.kernel.active) psfMatchAL = ipDiffim.ImagePsfMatchTask(config=confake) spatialSolution, psfMatchingKernel, backgroundModel = psfMatchAL._solve( kcs, basisList) fitCoeffs = psfMatchingKernel.getSpatialParameters() for b in range(len(fakeCoeffs)): for s in range(len(fakeCoeffs[b])): if fakeCoeffs[b][s] == 0.0: self.assertAlmostEqual(fitCoeffs[b][s], 0.0) else: # OUTSTANDING ISSUE - WHY IS THIS ONE TERM OFF!?!? if b != 4 and s != 0: self.assertAlmostEqual( fitCoeffs[b][s] / fakeCoeffs[b][s], 1.0, 1)
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "AL" self.subconfig = self.config.kernel.active # Test was put together before the min size went to 21 self.subconfig.kernelSize = 19 self.subconfig.scaleByFwhm = False self.subconfig.fitForBackground = True self.subconfig.spatialModelType = "polynomial" self.policy = pexConfig.makePolicy(self.subconfig) self.smi = afwImage.MaskedImageF('tests/compareToHotpants/scienceMI.fits') self.tmi = afwImage.MaskedImageF('tests/compareToHotpants/templateMI.fits') self.smi.setXY0(0, 0) self.tmi.setXY0(0, 0) # Run detection # detConfig = self.subconfig.detectionConfig # Note here regarding detConfig: # # If I set detThresholdType = "pixel_stdev", I get slightly # different centroids than if I use "stdev". These different # centroids screw up the testing since hotpants was hardcoded to # use the "stdev" centroids. For completeness these are: # # 32 32 # 96 32 # 160 32 # 96 95 # 31 96 # 160 96 # 96 160 # 160 160 # 32 160 # As of Winter2013, KernelCandidateDetectionF does not return # these exact centroids anymore, so I need to hardcode them # in. self.footprints = [] for xc, yc in [(32, 32), (96, 32), (160, 32), (96, 95), (31, 96), (160, 96), (96, 160), (160, 160), (32, 160)]: self.footprints.append(afwDet.Footprint(afwGeom.SpanSet( afwGeom.Box2I(afwGeom.Point2I(xc, yc), afwGeom.Extent2I(1, 1))))) # Make a basis list that hotpants has been run with nGauss = 1 sGauss = [3.] dGauss = [3] self.subconfig.alardNGauss = nGauss self.subconfig.alardSigGauss = sGauss self.subconfig.alardDegGauss = dGauss basisList0 = ipDiffim.makeKernelBasisList(self.subconfig) # HP does things in a different order, and with different normalization, so reorder list order = [0, 2, 5, 9, 1, 4, 8, 3, 7, 6] scaling = [1.000000e+00, 8.866037e-02, 1.218095e+01, 5.099318e-03, 8.866037e-02, 4.179772e-02, 1.138120e-02, 1.218095e+01, 1.138120e-02, 5.099318e-03] self.basisList = [] for i in range(len(order)): im = afwImage.ImageD(basisList0[order[i]].getDimensions()) basisList0[order[i]].computeImage(im, False) im /= scaling[i] # im.writeFits('k%d.fits' % (i)) k = afwMath.FixedKernel(im) self.basisList.append(k) # And a place to put candidates self.kernelCellSet = afwMath.SpatialCellSet(afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(self.smi.getWidth(), self.smi.getHeight())), self.policy.getInt("sizeCellX"), self.policy.getInt("sizeCellY")) # There are some -1 factors that come from differences in how # convolution is done. Some resulting convovled images end up # being a factor of -1 different, therefore the coefficients # need to be a factor of -1 different as well. self.parity = [1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1]
def setUp(self): self.configAL = ipDiffim.ImagePsfMatchTask.ConfigClass() self.configAL.kernel.name = "AL" self.subconfigAL = self.configAL.kernel.active self.configDF = ipDiffim.ImagePsfMatchTask.ConfigClass() self.configDF.kernel.name = "DF" self.subconfigDF = self.configDF.kernel.active self.configDFr = ipDiffim.ImagePsfMatchTask.ConfigClass() self.configDFr.kernel.name = "DF" self.subconfigDFr = self.configDFr.kernel.active self.subconfigDF.useRegularization = False self.subconfigDFr.useRegularization = True self.subconfigDFr.lambdaValue = 1000.0 self.subconfigAL.fitForBackground = fitForBackground self.subconfigDF.fitForBackground = fitForBackground self.subconfigDFr.fitForBackground = fitForBackground self.subconfigAL.constantVarianceWeighting = constantVarianceWeighting self.subconfigDF.constantVarianceWeighting = constantVarianceWeighting self.subconfigDFr.constantVarianceWeighting = constantVarianceWeighting self.kListAL = ipDiffim.makeKernelBasisList(self.subconfigAL) self.kListDF = ipDiffim.makeKernelBasisList(self.subconfigDF) self.kListDFr = ipDiffim.makeKernelBasisList(self.subconfigDFr) self.hMatDFr = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfigDFr)) self.bskvAL = ipDiffim.BuildSingleKernelVisitorF(self.kListAL, pexConfig.makePolicy(self.subconfigAL)) self.bskvDF = ipDiffim.BuildSingleKernelVisitorF(self.kListDF, pexConfig.makePolicy(self.subconfigDF)) self.bskvDFr = ipDiffim.BuildSingleKernelVisitorF(self.kListDFr, pexConfig.makePolicy(self.subconfigDF), self.hMatDFr) defSciencePath = globals()['defSciencePath'] defTemplatePath = globals()['defTemplatePath'] if defSciencePath and defTemplatePath: self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) else: defDataDir = lsst.utils.getPackageDir('afwdata') defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0", "v26-e0-c011-a00.sci.fits") defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci.fits") self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) warper = afwMath.Warper.fromConfig(self.subconfigAL.warpingConfig) self.templateExposure = warper.warpExposure(self.scienceExposure.getWcs(), self.templateExposure, destBBox=self.scienceExposure.getBBox()) # tmi = self.templateExposure.getMaskedImage() smi = self.scienceExposure.getMaskedImage() # Object detection detConfig = self.subconfigAL.detectionConfig detPolicy = pexConfig.makePolicy(detConfig) detPolicy.set("detThreshold", 50.) detPolicy.set("detThresholdType", "stdev") detPolicy.set("detOnTemplate", False) kcDetect = ipDiffim.KernelCandidateDetectionF(detPolicy) kcDetect.apply(tmi, smi) self.footprints = kcDetect.getFootprints()
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = long(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: print templateIdList template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution ctr = afwGeom.Box2D(exposure.getBBox()).getCenter() psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr)) scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) # sigma of PSF of template image before warping ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter() psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr)) templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn("Src product does not exist; running detection, measurement, selection") # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma = scienceSigmaPost, doSmooth = not self.doPreConvolve, idFactory = idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len(makeKernelBasisList(self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds, False) else: raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars(exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [k for i,k in enumerate(kernelSources) if i % self.config.controlStepSize] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory ) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId() : x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [m.first.get(refCoordKey) for m in wcsResults.matches] residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches] colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) + 2.5*numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys()]) dlong = numpy.array([r[0].asArcseconds() for s,r in zip(sids, cresiduals) if s in srcToMatch.keys()]) dlat = numpy.array([r[1].asArcseconds() for s,r in zip(sids, cresiduals) if s in srcToMatch.keys()]) idx1 = numpy.where(colors<self.sourceSelector.config.grMin) idx2 = numpy.where((colors>=self.sourceSelector.config.grMin)& (colors<=self.sourceSelector.config.grMax)) idx3 = numpy.where(colors>self.sourceSelector.config.grMax) rms1Long = IqrToSigma*(numpy.percentile(dlong[idx1],75)-numpy.percentile(dlong[idx1],25)) rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1],75)-numpy.percentile(dlat[idx1],25)) rms2Long = IqrToSigma*(numpy.percentile(dlong[idx2],75)-numpy.percentile(dlong[idx2],25)) rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2],75)-numpy.percentile(dlat[idx2],25)) rms3Long = IqrToSigma*(numpy.percentile(dlong[idx3],75)-numpy.percentile(dlong[idx3],25)) rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3],75)-numpy.percentile(dlat[idx3],25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference #Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister ) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Running diaSource detection") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve ) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") self.measurement.run(diaSources, subtractedExposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn("Src product does not exist; cannot match with diaSources") srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = measAstrom.AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = measAstrom.AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn("No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \ refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if srcMatchDict.has_key(sid): diaSource.set("srcMatchId", srcMatchDict[sid]) if refMatchDict.has_key(sid): diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cast_KernelCandidateF(cand)) # Get basis list to build control sample kernels basisList = afwMath.cast_LinearCombinationKernel( kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )
def run(self, sensorRef): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = long(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() if self.config.useWinter2013Hacks and self.config.winter2013borderMask > 0: self.log.warn("USING WINTER2013 HACK: MASKING BORDER PIXELS") bbox = exposure.getBBox(afwImage.PARENT) bbox.grow(-self.config.winter2013borderMask) self.setEdgeBits(exposure.getMaskedImage(), bbox, exposure.getMaskedImage().getMask().getPlaneBitMask("NO_DATA")) # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution ctr = afwGeom.Box2D(exposure.getBBox(afwImage.PARENT)).getCenter() psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr)) scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: templateExposure, templateSources = self.getTemplate(exposure, sensorRef) # sigma of PSF of template image before warping ctr = afwGeom.Box2D(templateExposure.getBBox(afwImage.PARENT)).getCenter() psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr)) templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT) # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getKernel().getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = psf afwMath.convolve(destMI, srcMI, preConvPsf.getKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn("Src product does not exist; running detection, measurement, selection") # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma = scienceSigmaPost, doSmooth = not self.doPreConvolve, idFactory = idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len(makeKernelBasisList(self.subtract.config.kernel.active, referenceFwhmPix = scienceSigmaPost * FwhmPerSigma, targetFwhmPix = templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources self.kcQa = diUtils.KernelCandidateQa(nparam, self.log) selectSources = self.kcQa.addToSchema(selectSources) astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig()) astromRet = astrometer.useKnownWcs(selectSources, exposure=exposure) matches = astromRet.matches kernelSources = self.sourceSelector.selectSources(exposure, selectSources, matches=matches) random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] [kernelSources.remove(x) for x in controlSources] self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" \ % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # First step: we need to subtract the background out # for detection and measurement. Use large binsize # for the background estimation. binsize = self.config.templateBgBinSize # Second step: we need to run detection on the # background-subtracted template # # Estimate FWHM for detection templateSources = self.subtract.getSelectSources( templateExposure, sigma = templateSigma, doSmooth = True, idFactory = idFactory, binsize = binsize, ) # Third step: we need to fit the relative astrometry. # # One problem is that the SIP fits are w.r.t. CRPIX, # and these coadd patches have the CRPIX of the entire # tract, i.e. off the image. This causes # register.fitWcs to fail. A workaround for now is to # re-fit the Wcs which returns with a CRPIX that is on # the image, and *then* to fit for the relative Wcs. # # Requires low Sip order to avoid overfitting # useWinter2013Hacks includes us using the deep calexp # as the template. In this case we don't need to # refit the Wcs. if not self.config.useWinter2013Hacks: sipOrder = self.config.templateSipOrder astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(sipOrder=sipOrder)) newWcs = astrometer.determineWcs(templateSources, templateExposure).getWcs() results = self.register.run(templateSources, newWcs, templateExposure.getBBox(afwImage.PARENT), selectSources) else: if self.config.winter2013WcsShift > 0.0: offset = afwGeom.Extent2D(self.config.winter2013WcsShift, self.config.winter2013WcsShift) cKey = templateSources[0].getTable().getCentroidKey() for source in templateSources: centroid = source.get(cKey) source.set(cKey, centroid+offset) elif self.config.winter2013WcsRms > 0.0: cKey = templateSources[0].getTable().getCentroidKey() for source in templateSources: offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(), self.config.winter2013WcsRms*numpy.random.normal()) centroid = source.get(cKey) source.set(cKey, centroid+offset) results = self.register.run(templateSources, templateExposure.getWcs(), templateExposure.getBBox(afwImage.PARENT), selectSources) warpedExp = self.register.warpExposure(templateExposure, results.wcs, exposure.getWcs(), exposure.getBBox(afwImage.PARENT)) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: refCoordKey = results.matches[0].first.getTable().getCoordKey() inCentroidKey = results.matches[0].second.getTable().getCentroidKey() sids = [m.first.getId() for m in results.matches] positions = [m.first.get(refCoordKey) for m in results.matches] residuals = [m.first.get(refCoordKey).getOffsetFrom( results.wcs.pixelToSky(m.second.get(inCentroidKey))) for m in results.matches] allresids = dict(zip(sids, zip(positions, residuals))) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference #Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure = templateExposure, scienceExposure = exposure, scienceFwhmPix = scienceSigmaPost * FwhmPerSigma, templateFwhmPix = templateSigma * FwhmPerSigma, candidateList = kernelSources, convolveTemplate = self.config.convolveTemplate, doWarping = not self.config.doUseRegister ) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Running diaSource detection") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: templateExposure, templateSources = self.getTemplate(exposure, sensorRef) subtractedExposure.setPsf(templateExposure.getPsf()) # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table = table, exposure = subtractedExposure, doSmooth = not self.config.doPreConvolve ) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if len(diaSources) < self.config.maxDiaSourcesToMeasure: self.dipolemeasurement.run(subtractedExposure, diaSources) else: self.measurement.run(subtractedExposure, diaSources) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds() # This does not do what I expect so I cobbled together a brute force method in python srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for \ srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn("Src product does not exist; cannot match with diaSources") srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(catalogMatchDist=matchRadAsec)) astromRet = astrometer.useKnownWcs(diaSources, exposure=exposure) refMatches = astromRet.matches if refMatches is None: self.log.warn("No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \ refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if srcMatchDict.has_key(sid): diaSource.set("srcMatchId", srcMatchDict[sid]) if refMatchDict.has_key(sid): diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sourceWriteFlags = (0 if self.config.doWriteHeavyFootprintsInSources else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS) sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc", flags=sourceWriteFlags) if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cast_KernelCandidateF(cand)) # Get basis list to build control sample kernels basisList = afwMath.cast_LinearCombinationKernel( kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList() controlCandList = \ diffimTools.sourceTableToCandList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, dobuild=True, basisList=basisList) self.kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) self.kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: self.kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: self.kcQa.aggregate(selectSources, self.metadata, allresids) #Persist using butler sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure = subtractedExposure, subtractRes = subtractRes, sources = diaSources, )
def setUp(self): self.configAL = ipDiffim.ImagePsfMatchTask.ConfigClass() self.configAL.kernel.name = "AL" self.subconfigAL = self.configAL.kernel.active self.configDF = ipDiffim.ImagePsfMatchTask.ConfigClass() self.configDF.kernel.name = "DF" self.subconfigDF = self.configDF.kernel.active self.configDFr = ipDiffim.ImagePsfMatchTask.ConfigClass() self.configDFr.kernel.name = "DF" self.subconfigDFr = self.configDFr.kernel.active self.subconfigDF.useRegularization = False self.subconfigDFr.useRegularization = True self.subconfigDFr.lambdaValue = 1000.0 self.subconfigAL.fitForBackground = fitForBackground self.subconfigDF.fitForBackground = fitForBackground self.subconfigDFr.fitForBackground = fitForBackground self.subconfigAL.constantVarianceWeighting = constantVarianceWeighting self.subconfigDF.constantVarianceWeighting = constantVarianceWeighting self.subconfigDFr.constantVarianceWeighting = constantVarianceWeighting self.kListAL = ipDiffim.makeKernelBasisList(self.subconfigAL) self.kListDF = ipDiffim.makeKernelBasisList(self.subconfigDF) self.kListDFr = ipDiffim.makeKernelBasisList(self.subconfigDFr) self.hMatDFr = ipDiffim.makeRegularizationMatrix( pexConfig.makePropertySet(self.subconfigDFr)) self.bskvAL = ipDiffim.BuildSingleKernelVisitorF( self.kListAL, pexConfig.makePropertySet(self.subconfigAL)) self.bskvDF = ipDiffim.BuildSingleKernelVisitorF( self.kListDF, pexConfig.makePropertySet(self.subconfigDF)) self.bskvDFr = ipDiffim.BuildSingleKernelVisitorF( self.kListDFr, pexConfig.makePropertySet(self.subconfigDF), self.hMatDFr) defSciencePath = globals()['defSciencePath'] defTemplatePath = globals()['defTemplatePath'] if defSciencePath and defTemplatePath: self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) else: defDataDir = lsst.utils.getPackageDir('afwdata') defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0", "v26-e0-c011-a00.sci.fits") defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci.fits") self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) warper = afwMath.Warper.fromConfig(self.subconfigAL.warpingConfig) self.templateExposure = warper.warpExposure( self.scienceExposure.getWcs(), self.templateExposure, destBBox=self.scienceExposure.getBBox()) # tmi = self.templateExposure.getMaskedImage() smi = self.scienceExposure.getMaskedImage() # Object detection detConfig = self.subconfigAL.detectionConfig detps = pexConfig.makePropertySet(detConfig) detps["detThreshold"] = 50. detps["detThresholdType"] = "stdev" detps["detOnTemplate"] = False kcDetect = ipDiffim.KernelCandidateDetectionF(detps) kcDetect.apply(tmi, smi) self.footprints = kcDetect.getFootprints()
def testMakeKernelBasisList(self): ks = ipDiffim.makeKernelBasisList(self.subconfigAL) self.alardLuptonTest(ks) ks = ipDiffim.makeKernelBasisList(self.subconfigDF) self.deltaFunctionTest(ks)
kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception, e: pass else: self.fail() try: kc.getImage() except Exception: pass else: self.fail() kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3): if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True) sIm *= scaling kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy)
kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception, e: pass else: self.fail() try: kc.getImage() except Exception: pass else: self.fail() kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True) def testDeltaFunctionScaled(self, scaling = 2.7, bg = 11.3): if not self.defDataDir: print >> sys.stderr, "Warning: afwdata is not set up" return sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True) sIm *= scaling kc = ipDiffim.KernelCandidateF(self.x02, self.y02, self.templateExposure2.getMaskedImage(), sIm, self.policy)
def testSourceConstructor(self): source = self.ss.addNew() source.setId(1) source.set(self.table.getCentroidSlot().getMeasKey().getX(), 276) source.set(self.table.getCentroidSlot().getMeasKey().getY(), 717) source.set("slot_PsfFlux_instFlux", 1.) kc = ipDiffim.KernelCandidateF(source, self.templateExposure2.getMaskedImage(), self.scienceImage2.getMaskedImage(), self.ps) # Kernel not initialized self.assertEqual(kc.isInitialized(), False) # Check that the source is set self.assertEqual(kc.getSource(), source) self.assertEqual(kc.getCandidateRating(), source.getPsfInstFlux()) # But this should be set on construction try: kc.getCandidateRating() except Exception as e: print(e) self.fail() # And these should be filled try: kc.getTemplateMaskedImage() kc.getScienceMaskedImage() except Exception as e: print(e) self.fail() # And of the right type self.assertEqual(type(kc.getTemplateMaskedImage()), afwImage.MaskedImageF) self.assertEqual(type(kc.getScienceMaskedImage()), afwImage.MaskedImageF) # None of these should work for kType in (ipDiffim.KernelCandidateF.ORIG, ipDiffim.KernelCandidateF.PCA, ipDiffim.KernelCandidateF.RECENT): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception: pass else: self.fail() try: kc.getImage() except Exception: pass else: self.fail() kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True)
def run(self, sensorRef, templateIdList=None): """Subtract an image from a template coadd and measure the result Steps include: - warp template coadd to match WCS of image - PSF match image to warped template - subtract image from PSF-matched, warped template - persist difference image - detect sources - measure sources @param sensorRef: sensor-level butler data reference, used for the following data products: Input only: - calexp - psf - ccdExposureId - ccdExposureId_bits - self.config.coaddName + "Coadd_skyMap" - self.config.coaddName + "Coadd" Input or output, depending on config: - self.config.coaddName + "Diff_subtractedExp" Output, depending on config: - self.config.coaddName + "Diff_matchedExp" - self.config.coaddName + "Diff_src" @return pipe_base Struct containing these fields: - subtractedExposure: exposure after subtracting template; the unpersisted version if subtraction not run but detection run None if neither subtraction nor detection run (i.e. nothing useful done) - subtractRes: results of subtraction task; None if subtraction not run - sources: detected and possibly measured sources; None if detection not run """ self.log.info("Processing %s" % (sensorRef.dataId)) # initialize outputs and some intermediate products subtractedExposure = None subtractRes = None selectSources = None kernelSources = None controlSources = None diaSources = None # We make one IdFactory that will be used by both icSrc and src datasets; # I don't know if this is the way we ultimately want to do things, but at least # this ensures the source IDs are fully unique. expBits = sensorRef.get("ccdExposureId_bits") expId = int(sensorRef.get("ccdExposureId")) idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits) # Retrieve the science image we wish to analyze exposure = sensorRef.get("calexp", immediate=True) if self.config.doAddCalexpBackground: mi = exposure.getMaskedImage() mi += sensorRef.get("calexpBackground").getImage() if not exposure.hasPsf(): raise pipeBase.TaskError("Exposure has no psf") sciencePsf = exposure.getPsf() subtractedExposureName = self.config.coaddName + "Diff_differenceExp" templateExposure = None # Stitched coadd exposure templateSources = None # Sources on the template image if self.config.doSubtract: template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList) templateExposure = template.exposure templateSources = template.sources # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius() # sigma of PSF of template image before warping templateSigma = templateExposure.getPsf().computeShape( ).getDeterminantRadius() # if requested, convolve the science exposure with its PSF # (properly, this should be a cross-correlation, but our code does not yet support that) # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done, # else sigma of original science exposure if self.config.doPreConvolve: convControl = afwMath.ConvolutionControl() # cannot convolve in place, so make a new MI to receive convolved image srcMI = exposure.getMaskedImage() destMI = srcMI.Factory(srcMI.getDimensions()) srcPsf = sciencePsf if self.config.useGaussianForPreConvolution: # convolve with a simplified PSF model: a double Gaussian kWidth, kHeight = sciencePsf.getLocalKernel( ).getDimensions() preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig) else: # convolve with science exposure's PSF model preConvPsf = srcPsf afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl) exposure.setMaskedImage(destMI) scienceSigmaPost = scienceSigmaOrig * math.sqrt(2) else: scienceSigmaPost = scienceSigmaOrig # If requested, find sources in the image if self.config.doSelectSources: if not sensorRef.datasetExists("src"): self.log.warn( "Src product does not exist; running detection, measurement, selection" ) # Run own detection and measurement; necessary in nightly processing selectSources = self.subtract.getSelectSources( exposure, sigma=scienceSigmaPost, doSmooth=not self.doPreConvolve, idFactory=idFactory, ) else: self.log.info("Source selection via src product") # Sources already exist; for data release processing selectSources = sensorRef.get("src") # Number of basis functions nparam = len( makeKernelBasisList( self.subtract.config.kernel.active, referenceFwhmPix=scienceSigmaPost * FwhmPerSigma, targetFwhmPix=templateSigma * FwhmPerSigma)) if self.config.doAddMetrics: # Modify the schema of all Sources kcQa = KernelCandidateQa(nparam) selectSources = kcQa.addToSchema(selectSources) if self.config.kernelSourcesFromRef: # match exposure sources to reference catalog astromRet = self.astrometer.loadAndMatch( exposure=exposure, sourceCat=selectSources) matches = astromRet.matches elif templateSources: # match exposure sources to template sources mc = afwTable.MatchControl() mc.findOnlyClosest = False matches = afwTable.matchRaDec(templateSources, selectSources, 1.0 * afwGeom.arcseconds, mc) else: raise RuntimeError( "doSelectSources=True and kernelSourcesFromRef=False," + "but template sources not available. Cannot match science " + "sources with template sources. Run process* on data from " + "which templates are built.") kernelSources = self.sourceSelector.selectStars( exposure, selectSources, matches=matches).starCat random.shuffle(kernelSources, random.random) controlSources = kernelSources[::self.config.controlStepSize] kernelSources = [ k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize ] if self.config.doSelectDcrCatalog: redSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=self.sourceSelector.config.grMax, grMax=99.999)) redSources = redSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(redSources) blueSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig( grMin=-99.999, grMax=self.sourceSelector.config.grMin)) blueSources = blueSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(blueSources) if self.config.doSelectVariableCatalog: varSelector = DiaCatalogSourceSelectorTask( DiaCatalogSourceSelectorConfig(includeVariable=True)) varSources = varSelector.selectStars( exposure, selectSources, matches=matches).starCat controlSources.extend(varSources) self.log.info( "Selected %d / %d sources for Psf matching (%d for control sample)" % (len(kernelSources), len(selectSources), len(controlSources))) allresids = {} if self.config.doUseRegister: self.log.info("Registering images") if templateSources is None: # Run detection on the template, which is # temporarily background-subtracted templateSources = self.subtract.getSelectSources( templateExposure, sigma=templateSigma, doSmooth=True, idFactory=idFactory) # Third step: we need to fit the relative astrometry. # wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources) warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs, exposure.getWcs(), exposure.getBBox()) templateExposure = warpedExp # Create debugging outputs on the astrometric # residuals as a function of position. Persistence # not yet implemented; expected on (I believe) #2636. if self.config.doDebugRegister: # Grab matches to reference catalog srcToMatch = {x.second.getId(): x.first for x in matches} refCoordKey = wcsResults.matches[0].first.getTable( ).getCoordKey() inCentroidKey = wcsResults.matches[0].second.getTable( ).getCentroidKey() sids = [m.first.getId() for m in wcsResults.matches] positions = [ m.first.get(refCoordKey) for m in wcsResults.matches ] residuals = [ m.first.get(refCoordKey).getOffsetFrom( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] allresids = dict(zip(sids, zip(positions, residuals))) cresiduals = [ m.first.get(refCoordKey).getTangentPlaneOffset( wcsResults.wcs.pixelToSky( m.second.get(inCentroidKey))) for m in wcsResults.matches ] colors = numpy.array([ -2.5 * numpy.log10(srcToMatch[x].get("g")) + 2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids if x in srcToMatch.keys() ]) dlong = numpy.array([ r[0].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) dlat = numpy.array([ r[1].asArcseconds() for s, r in zip(sids, cresiduals) if s in srcToMatch.keys() ]) idx1 = numpy.where( colors < self.sourceSelector.config.grMin) idx2 = numpy.where( (colors >= self.sourceSelector.config.grMin) & (colors <= self.sourceSelector.config.grMax)) idx3 = numpy.where( colors > self.sourceSelector.config.grMax) rms1Long = IqrToSigma * \ (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25)) rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) - numpy.percentile(dlat[idx1], 25)) rms2Long = IqrToSigma * \ (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25)) rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) - numpy.percentile(dlat[idx2], 25)) rms3Long = IqrToSigma * \ (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25)) rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) - numpy.percentile(dlat[idx3], 25)) self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]), rms1Long, numpy.median(dlat[idx1]), rms1Lat)) self.log.info( "Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]), rms2Long, numpy.median(dlat[idx2]), rms2Lat)) self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]), rms3Long, numpy.median(dlat[idx3]), rms3Lat)) self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1])) self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2])) self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3])) self.metadata.add("RegisterBlueLongOffsetStd", rms1Long) self.metadata.add("RegisterGreenLongOffsetStd", rms2Long) self.metadata.add("RegisterRedLongOffsetStd", rms3Long) self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1])) self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2])) self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3])) self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat) self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat) self.metadata.add("RegisterRedLatOffsetStd", rms3Lat) # warp template exposure to match exposure, # PSF match template exposure to exposure, # then return the difference # Return warped template... Construct sourceKernelCand list after subtract self.log.info("Subtracting images") subtractRes = self.subtract.subtractExposures( templateExposure=templateExposure, scienceExposure=exposure, candidateList=kernelSources, convolveTemplate=self.config.convolveTemplate, doWarping=not self.config.doUseRegister) subtractedExposure = subtractRes.subtractedExposure if self.config.doWriteMatchedExp: sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp") if self.config.doDetection: self.log.info("Computing diffim PSF") if subtractedExposure is None: subtractedExposure = sensorRef.get(subtractedExposureName) # Get Psf from the appropriate input image if it doesn't exist if not subtractedExposure.hasPsf(): if self.config.convolveTemplate: subtractedExposure.setPsf(exposure.getPsf()) else: if templateExposure is None: template = self.getTemplate.run( exposure, sensorRef, templateIdList=templateIdList) subtractedExposure.setPsf(template.exposure.getPsf()) # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False. if self.config.doDecorrelation and self.config.doSubtract: decorrResult = self.decorrelate.run(exposure, templateExposure, subtractedExposure, subtractRes.psfMatchingKernel) subtractedExposure = decorrResult.correctedExposure if self.config.doDetection: self.log.info("Running diaSource detection") # Erase existing detection mask planes mask = subtractedExposure.getMaskedImage().getMask() mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE")) table = afwTable.SourceTable.make(self.schema, idFactory) table.setMetadata(self.algMetadata) results = self.detection.makeSourceCatalog( table=table, exposure=subtractedExposure, doSmooth=not self.config.doPreConvolve) if self.config.doMerge: fpSet = results.fpSets.positive fpSet.merge(results.fpSets.negative, self.config.growFootprint, self.config.growFootprint, False) diaSources = afwTable.SourceCatalog(table) fpSet.makeSources(diaSources) self.log.info("Merging detections into %d sources" % (len(diaSources))) else: diaSources = results.sources if self.config.doMeasurement: self.log.info("Running diaSource measurement") if not self.config.doDipoleFitting: self.measurement.run(diaSources, subtractedExposure) else: if self.config.doSubtract: self.measurement.run(diaSources, subtractedExposure, exposure, subtractRes.matchedExposure) else: self.measurement.run(diaSources, subtractedExposure, exposure) # Match with the calexp sources if possible if self.config.doMatchSources: if sensorRef.datasetExists("src"): # Create key,val pair where key=diaSourceId and val=sourceId matchRadAsec = self.config.diaSourceMatchRadius matchRadPixel = matchRadAsec / exposure.getWcs( ).pixelScale().asArcseconds() srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel) srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for srcMatch in srcMatches]) self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict), len(diaSources))) else: self.log.warn( "Src product does not exist; cannot match with diaSources" ) srcMatchDict = {} # Create key,val pair where key=diaSourceId and val=refId refAstromConfig = AstrometryConfig() refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec refAstrometer = AstrometryTask(refAstromConfig) astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources) refMatches = astromRet.matches if refMatches is None: self.log.warn( "No diaSource matches with reference catalog") refMatchDict = {} else: self.log.info( "Matched %d / %d diaSources to reference catalog" % (len(refMatches), len(diaSources))) refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for refMatch in refMatches]) # Assign source Ids for diaSource in diaSources: sid = diaSource.getId() if sid in srcMatchDict: diaSource.set("srcMatchId", srcMatchDict[sid]) if sid in refMatchDict: diaSource.set("refMatchId", refMatchDict[sid]) if diaSources is not None and self.config.doWriteSources: sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc") if self.config.doAddMetrics and self.config.doSelectSources: self.log.info("Evaluating metrics and control sample") kernelCandList = [] for cell in subtractRes.kernelCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates kernelCandList.append(cand) # Get basis list to build control sample kernels basisList = kernelCandList[0].getKernel( KernelCandidateF.ORIG).getKernelList() controlCandList = \ diffimTools.sourceTableToCandidateList(controlSources, subtractRes.warpedExposure, exposure, self.config.subtract.kernel.active, self.config.subtract.kernel.active.detectionConfig, self.log, doBuild=True, basisList=basisList) kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel, dof=nparam) kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel) if self.config.doDetection: kcQa.aggregate(selectSources, self.metadata, allresids, diaSources) else: kcQa.aggregate(selectSources, self.metadata, allresids) sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc") if self.config.doWriteSubtractedExp: sensorRef.put(subtractedExposure, subtractedExposureName) self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources) return pipeBase.Struct( subtractedExposure=subtractedExposure, subtractRes=subtractRes, sources=diaSources, )
def setUp(self): self.config = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config.kernel.name = "AL" self.subconfig = self.config.kernel.active # Test was put together before the min size went to 21 self.subconfig.kernelSize = 19 self.subconfig.scaleByFwhm = False self.subconfig.fitForBackground = True self.subconfig.spatialModelType = "polynomial" self.ps = pexConfig.makePropertySet(self.subconfig) self.smi = afwImage.MaskedImageF( 'tests/compareToHotpants/scienceMI.fits') self.tmi = afwImage.MaskedImageF( 'tests/compareToHotpants/templateMI.fits') self.smi.setXY0(0, 0) self.tmi.setXY0(0, 0) # Run detection # detConfig = self.subconfig.detectionConfig # Note here regarding detConfig: # # If I set detThresholdType = "pixel_stdev", I get slightly # different centroids than if I use "stdev". These different # centroids screw up the testing since hotpants was hardcoded to # use the "stdev" centroids. For completeness these are: # # 32 32 # 96 32 # 160 32 # 96 95 # 31 96 # 160 96 # 96 160 # 160 160 # 32 160 # As of Winter2013, KernelCandidateDetectionF does not return # these exact centroids anymore, so I need to hardcode them # in. self.footprints = [] for xc, yc in [(32, 32), (96, 32), (160, 32), (96, 95), (31, 96), (160, 96), (96, 160), (160, 160), (32, 160)]: self.footprints.append( afwDet.Footprint( afwGeom.SpanSet( geom.Box2I(geom.Point2I(xc, yc), geom.Extent2I(1, 1))))) # Make a basis list that hotpants has been run with nGauss = 1 sGauss = [3.] dGauss = [3] self.subconfig.alardNGauss = nGauss self.subconfig.alardSigGauss = sGauss self.subconfig.alardDegGauss = dGauss basisList0 = ipDiffim.makeKernelBasisList(self.subconfig) # HP does things in a different order, and with different normalization, so reorder list order = [0, 2, 5, 9, 1, 4, 8, 3, 7, 6] scaling = [ 1.000000e+00, 8.866037e-02, 1.218095e+01, 5.099318e-03, 8.866037e-02, 4.179772e-02, 1.138120e-02, 1.218095e+01, 1.138120e-02, 5.099318e-03 ] self.basisList = [] for i in range(len(order)): im = afwImage.ImageD(basisList0[order[i]].getDimensions()) basisList0[order[i]].computeImage(im, False) im /= scaling[i] # im.writeFits('k%d.fits' % (i)) k = afwMath.FixedKernel(im) self.basisList.append(k) # And a place to put candidates self.kernelCellSet = afwMath.SpatialCellSet( geom.Box2I( geom.Point2I(0, 0), geom.Extent2I(self.smi.getWidth(), self.smi.getHeight())), self.ps["sizeCellX"], self.ps["sizeCellY"]) # There are some -1 factors that come from differences in how # convolution is done. Some resulting convovled images end up # being a factor of -1 different, therefore the coefficients # need to be a factor of -1 different as well. self.parity = [1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1]
def setUp(self, CFHT=True): lambdaValue = 1.0 self.config1 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config1.kernel.name = "DF" self.subconfig1 = self.config1.kernel.active self.config2 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config2.kernel.name = "DF" self.subconfig2 = self.config2.kernel.active self.config3 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config3.kernel.name = "DF" self.subconfig3 = self.config3.kernel.active self.config4 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config4.kernel.name = "DF" self.subconfig4 = self.config4.kernel.active self.subconfig1.useRegularization = False self.subconfig2.useRegularization = True self.subconfig2.lambdaType = "absolute" self.subconfig2.lambdaValue = lambdaValue self.subconfig2.regularizationType = "centralDifference" self.subconfig2.centralRegularizationStencil = 5 self.subconfig3.useRegularization = True self.subconfig3.lambdaType = "absolute" self.subconfig3.lambdaValue = lambdaValue self.subconfig3.regularizationType = "centralDifference" self.subconfig3.centralRegularizationStencil = 9 self.subconfig4.useRegularization = True self.subconfig4.lambdaType = "absolute" self.subconfig4.lambdaValue = lambdaValue self.subconfig4.regularizationType = "forwardDifference" self.subconfig4.forwardRegularizationOrders = [1, 2] self.kList1 = ipDiffim.makeKernelBasisList(self.subconfig1) self.bskv1 = ipDiffim.BuildSingleKernelVisitorF( self.kList1, pexConfig.makePropertySet(self.subconfig1)) self.kList2 = ipDiffim.makeKernelBasisList(self.subconfig2) self.hMat2 = ipDiffim.makeRegularizationMatrix( pexConfig.makePropertySet(self.subconfig2)) self.bskv2 = ipDiffim.BuildSingleKernelVisitorF( self.kList2, pexConfig.makePropertySet(self.subconfig2), self.hMat2) self.kList3 = ipDiffim.makeKernelBasisList(self.subconfig3) self.hMat3 = ipDiffim.makeRegularizationMatrix( pexConfig.makePropertySet(self.subconfig3)) self.bskv3 = ipDiffim.BuildSingleKernelVisitorF( self.kList3, pexConfig.makePropertySet(self.subconfig3), self.hMat3) self.kList4 = ipDiffim.makeKernelBasisList(self.subconfig4) self.hMat4 = ipDiffim.makeRegularizationMatrix( pexConfig.makePropertySet(self.subconfig4)) self.bskv4 = ipDiffim.BuildSingleKernelVisitorF( self.kList4, pexConfig.makePropertySet(self.subconfig4), self.hMat4) # known input images defDataDir = lsst.utils.getPackageDir('afwdata') if CFHT: defSciencePath = os.path.join(defDataDir, 'CFHT', 'D4', CFHTTORUN + '.fits') defTemplatePath = os.path.join(defDataDir, 'CFHT', 'D4', CFHTTORUN + '_tmpl.fits') # no need to remap self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) else: defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0", "v26-e0-c011-a00.sci") defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci") self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) warper = afwMath.Warper.fromConfig(self.subconfig1.warpingConfig) self.templateExposure = warper.warpExposure( self.scienceExposure.getWcs(), self.templateExposure, destBBox=self.scienceExposure.getBBox()) diffimTools.backgroundSubtract(self.subconfig1.afwBackgroundConfig, [ self.scienceExposure.getMaskedImage(), self.templateExposure.getMaskedImage() ]) # tmi = self.templateExposure.getMaskedImage() smi = self.scienceExposure.getMaskedImage() detConfig = self.subconfig1.detectionConfig detps = pexConfig.makePropertySet(detConfig) detps["detThreshold"] = 50. detps["detOnTemplate"] = False kcDetect = ipDiffim.KernelCandidateDetectionF(detps) kcDetect.apply(tmi, smi) self.footprints = kcDetect.getFootprints()
def testSourceConstructor(self): source = self.ss.addNew() source.setId(1) source.set(self.table.getCentroidKey().getX(), 276) source.set(self.table.getCentroidKey().getY(), 717) source.set("slot_PsfFlux_instFlux", 1.) kc = ipDiffim.KernelCandidateF(source, self.templateExposure2.getMaskedImage(), self.scienceImage2.getMaskedImage(), self.policy) # Kernel not initialized self.assertEqual(kc.isInitialized(), False) # Check that the source is set self.assertEqual(kc.getSource(), source) self.assertEqual(kc.getCandidateRating(), source.getPsfInstFlux()) # But this should be set on construction try: kc.getCandidateRating() except Exception as e: print(e) self.fail() # And these should be filled try: kc.getTemplateMaskedImage() kc.getScienceMaskedImage() except Exception as e: print(e) self.fail() # And of the right type self.assertEqual(type(kc.getTemplateMaskedImage()), afwImage.MaskedImageF) self.assertEqual(type(kc.getScienceMaskedImage()), afwImage.MaskedImageF) # None of these should work for kType in (ipDiffim.KernelCandidateF.ORIG, ipDiffim.KernelCandidateF.PCA, ipDiffim.KernelCandidateF.RECENT): for kMethod in (kc.getKernelSolution, kc.getKernel, kc.getBackground, kc.getKsum, kc.getKernelImage, kc.getDifferenceImage): try: kMethod(kType) except Exception: pass else: self.fail() try: kc.getImage() except Exception: pass else: self.fail() kList = ipDiffim.makeKernelBasisList(self.subconfig) kc.build(kList) self.assertEqual(kc.isInitialized(), True)
fnum = 1 for switch in ['A', 'B', 'C']: if switch == 'A': # AL config = subconfigAL elif switch == 'B': # AL with ~320 bases config = subconfigAL config.alardDegGauss = (15, 10, 5) elif switch == 'C': config = subconfigDF config.useRegularization = False kList = ipDiffim.makeKernelBasisList(config) policy = pexConfig.makePolicy(config) bskv = ipDiffim.BuildSingleKernelVisitorF(kList, policy) # TEST 1 tmi, smi = makeTest1(doAddNoise) kc = ipDiffim.makeKernelCandidate(0.0, 0.0, tmi, smi, policy) bskv.processCandidate(kc) kernel = kc.getKernel(ipDiffim.KernelCandidateF.ORIG) kimage = afwImage.ImageD(kernel.getDimensions()) kernel.computeImage(kimage, False) diffim = kc.getDifferenceImage(ipDiffim.KernelCandidateF.ORIG) afwDisplay.Display(frame=fnum).mtv(tmi)
fnum = 1 for switch in ['A', 'B', 'C']: if switch == 'A': # Default Alard Lupton config = subconfigAL elif switch == 'B': # Add more AL bases (typically 4 3 2) config = subconfigAL config.alardDegGauss = (8, 6, 4) elif switch == 'C': # Delta function config = subconfigDF config.useRegularization = False kList = ipDiffim.makeKernelBasisList(config) policy = pexConfig.makePolicy(config) bskv = ipDiffim.BuildSingleKernelVisitorF(kList, policy) # TEST 1 tmi, smi = makeTest1(doAddNoise) kc = ipDiffim.makeKernelCandidate(0.0, 0.0, tmi, smi, policy) bskv.processCandidate(kc) kernel = kc.getKernel(ipDiffim.KernelCandidateF.ORIG) kimage = afwImage.ImageD(kernel.getDimensions()) kernel.computeImage(kimage, False) diffim = kc.getDifferenceImage(ipDiffim.KernelCandidateF.ORIG) ds9.mtv(tmi, frame=fnum)
def testMakeKernelBasisList(self): ks = ipDiffim.makeKernelBasisList(self.subconfigAL) self.alardLuptonTest(ks) ks = ipDiffim.makeKernelBasisList(self.subconfigDF) self.deltaFunctionTest(ks)
def setUp(self, CFHT=True): lambdaValue = 1.0 self.config1 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config1.kernel.name = "DF" self.subconfig1 = self.config1.kernel.active self.config2 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config2.kernel.name = "DF" self.subconfig2 = self.config2.kernel.active self.config3 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config3.kernel.name = "DF" self.subconfig3 = self.config3.kernel.active self.config4 = ipDiffim.ImagePsfMatchTask.ConfigClass() self.config4.kernel.name = "DF" self.subconfig4 = self.config4.kernel.active self.subconfig1.useRegularization = False self.subconfig2.useRegularization = True self.subconfig2.lambdaType = "absolute" self.subconfig2.lambdaValue = lambdaValue self.subconfig2.regularizationType = "centralDifference" self.subconfig2.centralRegularizationStencil = 5 self.subconfig3.useRegularization = True self.subconfig3.lambdaType = "absolute" self.subconfig3.lambdaValue = lambdaValue self.subconfig3.regularizationType = "centralDifference" self.subconfig3.centralRegularizationStencil = 9 self.subconfig4.useRegularization = True self.subconfig4.lambdaType = "absolute" self.subconfig4.lambdaValue = lambdaValue self.subconfig4.regularizationType = "forwardDifference" self.subconfig4.forwardRegularizationOrders = [1, 2] self.kList1 = ipDiffim.makeKernelBasisList(self.subconfig1) self.bskv1 = ipDiffim.BuildSingleKernelVisitorF(self.kList1, pexConfig.makePolicy(self.subconfig1)) self.kList2 = ipDiffim.makeKernelBasisList(self.subconfig2) self.hMat2 = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfig2)) self.bskv2 = ipDiffim.BuildSingleKernelVisitorF(self.kList2, pexConfig.makePolicy(self.subconfig2), self.hMat2) self.kList3 = ipDiffim.makeKernelBasisList(self.subconfig3) self.hMat3 = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfig3)) self.bskv3 = ipDiffim.BuildSingleKernelVisitorF(self.kList3, pexConfig.makePolicy(self.subconfig3), self.hMat3) self.kList4 = ipDiffim.makeKernelBasisList(self.subconfig4) self.hMat4 = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfig4)) self.bskv4 = ipDiffim.BuildSingleKernelVisitorF(self.kList4, pexConfig.makePolicy(self.subconfig4), self.hMat4) # known input images defDataDir = lsst.utils.getPackageDir('afwdata') if CFHT: defSciencePath = os.path.join(defDataDir, 'CFHT', 'D4', CFHTTORUN+'.fits') defTemplatePath = os.path.join(defDataDir, 'CFHT', 'D4', CFHTTORUN+'_tmpl.fits') # no need to remap self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) else: defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0", "v26-e0-c011-a00.sci") defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci") self.scienceExposure = afwImage.ExposureF(defSciencePath) self.templateExposure = afwImage.ExposureF(defTemplatePath) warper = afwMath.Warper.fromConfig(self.subconfig1.warpingConfig) self.templateExposure = warper.warpExposure(self.scienceExposure.getWcs(), self.templateExposure, destBBox=self.scienceExposure.getBBox()) diffimTools.backgroundSubtract(self.subconfig1.afwBackgroundConfig, [self.scienceExposure.getMaskedImage(), self.templateExposure.getMaskedImage()]) # tmi = self.templateExposure.getMaskedImage() smi = self.scienceExposure.getMaskedImage() detConfig = self.subconfig1.detectionConfig detPolicy = pexConfig.makePolicy(detConfig) detPolicy.set("detThreshold", 50.) detPolicy.set("detOnTemplate", False) kcDetect = ipDiffim.KernelCandidateDetectionF(detPolicy) kcDetect.apply(tmi, smi) self.footprints = kcDetect.getFootprints()