def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3):
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(),
deep=True)
sIm *= scaling
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm, self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(
ipDiffim.KernelCandidateF.RECENT),
kSum=scaling)
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(),
deep=True)
sIm += bg
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm, self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(
ipDiffim.KernelCandidateF.RECENT),
bg=bg)
def testGaussianWithNoise(self):
# Convolve a real image with a gaussian and try and recover
# it. Add noise and perform the same test.
gsize = self.policy.getInt("kernelSize")
gaussFunction = afwMath.GaussianFunction2D(2, 3)
gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction)
kImageIn = afwImage.ImageD(afwGeom.Extent2I(gsize, gsize))
kSumIn = gaussKernel.computeImage(kImageIn, False)
imX, imY = self.templateExposure2.getMaskedImage().getDimensions()
smi = afwImage.MaskedImageF(afwGeom.Extent2I(imX, imY))
afwMath.convolve(smi, self.templateExposure2.getMaskedImage(), gaussKernel, False)
bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL))
tmi2 = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), bbox, origin=afwImage.LOCAL)
smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL)
kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
kImageOut = kc.getImage()
soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)
self.assertAlmostEqual(soln.getKsum(), kSumIn)
# 8.7499380640430563e-06 != 0.0 within 7 places
self.assertAlmostEqual(soln.getBackground(), 0.0, 4)
for j in range(kImageOut.getHeight()):
for i in range(kImageOut.getWidth()):
# in the outskirts of the kernel, the ratio can get screwed because of low S/N
# e.g. 7.45817359824e-09 vs. 1.18062529402e-08
# in the guts of the kernel it should look closer
if kImageIn[i, j, afwImage.LOCAL] > 1e-4:
# sigh, too bad this sort of thing fails..
# 0.99941584433815966 != 1.0 within 3 places
self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL]/kImageIn[i, j, afwImage.LOCAL],
1.0, 2)
# now repeat with noise added; decrease precision of comparison
self.addNoise(smi2)
kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
kImageOut = kc.getImage()
soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)
self.assertAlmostEqual(soln.getKsum(), kSumIn, 3)
if not (self.policy.get("fitForBackground")):
self.assertEqual(soln.getBackground(), 0.0)
for j in range(kImageOut.getHeight()):
for i in range(kImageOut.getWidth()):
if kImageIn[i, j, afwImage.LOCAL] > 1e-2:
self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL],
kImageIn[i, j, afwImage.LOCAL], 2)
def testGaussianWithNoise(self):
# Convolve a real image with a gaussian and try and recover
# it. Add noise and perform the same test.
gsize = self.ps["kernelSize"]
gaussFunction = afwMath.GaussianFunction2D(2, 3)
gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction)
kImageIn = afwImage.ImageD(geom.Extent2I(gsize, gsize))
kSumIn = gaussKernel.computeImage(kImageIn, False)
imX, imY = self.templateExposure2.getMaskedImage().getDimensions()
smi = afwImage.MaskedImageF(geom.Extent2I(imX, imY))
afwMath.convolve(smi, self.templateExposure2.getMaskedImage(), gaussKernel, False)
bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL))
tmi2 = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), bbox, origin=afwImage.LOCAL)
smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL)
kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
kImageOut = kc.getImage()
soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)
self.assertAlmostEqual(soln.getKsum(), kSumIn)
# 8.7499380640430563e-06 != 0.0 within 7 places
self.assertAlmostEqual(soln.getBackground(), 0.0, 4)
for j in range(kImageOut.getHeight()):
for i in range(kImageOut.getWidth()):
# in the outskirts of the kernel, the ratio can get screwed because of low S/N
# e.g. 7.45817359824e-09 vs. 1.18062529402e-08
# in the guts of the kernel it should look closer
if kImageIn[i, j, afwImage.LOCAL] > 1e-4:
# sigh, too bad this sort of thing fails..
# 0.99941584433815966 != 1.0 within 3 places
self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL]/kImageIn[i, j, afwImage.LOCAL],
1.0, 2)
# now repeat with noise added; decrease precision of comparison
self.addNoise(smi2)
kc = ipDiffim.KernelCandidateF(self.x02, self.y02, tmi2, smi2, self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
kImageOut = kc.getImage()
soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)
self.assertAlmostEqual(soln.getKsum(), kSumIn, 3)
if not self.ps.get("fitForBackground"):
self.assertEqual(soln.getBackground(), 0.0)
for j in range(kImageOut.getHeight()):
for i in range(kImageOut.getWidth()):
if kImageIn[i, j, afwImage.LOCAL] > 1e-2:
self.assertAlmostEqual(kImageOut[i, j, afwImage.LOCAL],
kImageIn[i, j, afwImage.LOCAL], 2)
def testDeltaFunctionScaled(self, scaling = 2.7, bg = 11.3):
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True)
sIm *= scaling
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm,
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT),
kSum = scaling)
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True)
sIm += bg
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm,
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT),
bg = bg)
def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3):
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(),
True)
sIm *= scaling
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm, self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(
ipDiffim.KernelCandidateF.RECENT),
kSum=scaling)
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(),
True)
sIm += bg
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm, self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(
ipDiffim.KernelCandidateF.RECENT),
bg=bg)
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "DF"
self.subconfig = self.config.kernel.active
self.policy = pexConfig.makePolicy(self.subconfig)
self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
def testDeltaFunction(self):
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
# Match an image to itself, with delta-function basis set
# No regularization
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
self.templateExposure2.getMaskedImage(),
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
# These should work
for kType in (ipDiffim.KernelCandidateF.ORIG,
ipDiffim.KernelCandidateF.RECENT):
for kMethod in (kc.getKernelSolution,
kc.getKernel,
kc.getBackground,
kc.getKsum,
kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception, e:
print kMethod, e
self.fail()
else:
pass
def testDeltaFunction(self):
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
# Match an image to itself, with delta-function basis set
# No regularization
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
self.templateExposure2.getMaskedImage(),
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
# These should work
for kType in (ipDiffim.KernelCandidateF.ORIG,
ipDiffim.KernelCandidateF.RECENT):
for kMethod in (kc.getKernelSolution, kc.getKernel,
kc.getBackground, kc.getKsum, kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception, e:
print kMethod, e
self.fail()
else:
pass
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "DF"
self.subconfig = self.config.kernel.active
self.ps = pexConfig.makePropertySet(self.subconfig)
self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
def testSubtractMaskedImages(self):
# Lets do some additional testing here to make sure we recover
# the known spatial model. No background, just the faked
# alard-lupton basis set. The rest of matchMaskedImages() and
# subtractMaskedImages() functionality is tested by the
# Exposure-based methods.
fakeCoeffs = diffimTools.fakeCoeffs()
# Quick note; you shouldn't change confake here, since the
# candidates in the KernelCellSet are initialized in
# makeFakeKernelSet
tMi, sMi, sK, kcs, confake = diffimTools.makeFakeKernelSet(bgValue = 0.0, addNoise = False)
svar = sMi.getVariance()
svar.set(1.0)
tvar = tMi.getVariance()
tvar.set(1.0)
basisList = ipDiffim.makeKernelBasisList(confake.kernel.active)
psfMatchAL = ipDiffim.ImagePsfMatchTask(config=confake)
spatialSolution, psfMatchingKernel, backgroundModel = psfMatchAL._solve(kcs, basisList)
fitCoeffs = psfMatchingKernel.getSpatialParameters()
for b in range(len(fakeCoeffs)):
for s in range(len(fakeCoeffs[b])):
if fakeCoeffs[b][s] == 0.0:
self.assertAlmostEqual(fitCoeffs[b][s], 0.0)
else:
# OUTSTANDING ISSUE - WHY IS THIS ONE TERM OFF!?!?
if b != 4 and s != 0:
self.assertAlmostEqual(fitCoeffs[b][s]/fakeCoeffs[b][s], 1.0, 1)
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "DF"
self.subconfig = self.config.kernel.active
self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy = pexConfig.makePolicy(self.subconfig)
self.policy.set("useRegularization", False)
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "AL"
self.subconfig = self.config.kernel.active
self.policy = pexConfig.makePolicy(self.subconfig)
self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
self.ksize = self.policy.get('kernelSize')
def testDeltaFunction(self):
# Match an image to itself, with delta-function basis set
# No regularization
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
self.templateExposure2.getMaskedImage(),
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
# These should work
for kType in (ipDiffim.KernelCandidateF.ORIG,
ipDiffim.KernelCandidateF.RECENT):
for kMethod in (kc.getKernelSolution,
kc.getKernel,
kc.getBackground,
kc.getKsum,
kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception as e:
print(kMethod, e)
self.fail()
else:
pass
try:
kc.getImage()
except Exception as e:
print(kMethod, e)
self.fail()
else:
pass
# None of these should work
for kType in (ipDiffim.KernelCandidateF.PCA,):
for kMethod in (kc.getKernelSolution,
kc.getKernel,
kc.getBackground,
kc.getKsum,
kc.getKernelImage,
kc.getImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception:
pass
else:
print(kMethod)
self.fail()
self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT))
def testGaussian(self, imsize=50):
# Convolve a delta function with a known gaussian; try to
# recover using delta-function basis
gsize = self.ps["kernelSize"]
tsize = imsize + gsize
gaussFunction = afwMath.GaussianFunction2D(2, 3)
gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction)
kImageIn = afwImage.ImageD(geom.Extent2I(gsize, gsize))
gaussKernel.computeImage(kImageIn, False)
# template image with a single hot pixel in the exact center
tmi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize))
tmi.set(0, 0x0, 1e-4)
cpix = tsize // 2
tmi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 1)
# science image
smi = afwImage.MaskedImageF(tmi.getDimensions())
convolutionControl = afwMath.ConvolutionControl()
convolutionControl.setDoNormalize(False)
afwMath.convolve(smi, tmi, gaussKernel, convolutionControl)
# get the actual kernel sum (since the image is not infinite)
gscaling = afwMath.makeStatistics(smi,
afwMath.SUM).getValue(afwMath.SUM)
# grab only the non-masked subregion
bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL))
tmi2 = afwImage.MaskedImageF(tmi, bbox, origin=afwImage.LOCAL)
smi2 = afwImage.MaskedImageF(smi, bbox, origin=afwImage.LOCAL)
# make sure its a valid subregion!
for j in range(tmi2.getHeight()):
for i in range(tmi2.getWidth()):
self.assertEqual(tmi2.mask[i, j, afwImage.LOCAL], 0)
self.assertEqual(smi2.mask[i, j, afwImage.LOCAL], 0)
kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi2, smi2, self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
kImageOut = kc.getImage()
soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)
self.assertAlmostEqual(soln.getKsum(), gscaling)
self.assertAlmostEqual(soln.getBackground(), 0.0)
for j in range(kImageOut.getHeight()):
for i in range(kImageOut.getWidth()):
self.assertAlmostEqual(
kImageOut[i, j, afwImage.LOCAL] /
kImageIn[i, j, afwImage.LOCAL], 1.0, 5)
def testDeltaFunction(self):
# Match an image to itself, with delta-function basis set
# No regularization
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
self.templateExposure2.getMaskedImage(),
self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
# These should work
for kType in (ipDiffim.KernelCandidateF.ORIG,
ipDiffim.KernelCandidateF.RECENT):
for kMethod in (kc.getKernelSolution,
kc.getKernel,
kc.getBackground,
kc.getKsum,
kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception as e:
print(kMethod, e)
self.fail()
else:
pass
try:
kc.getImage()
except Exception as e:
print(kMethod, e)
self.fail()
else:
pass
# None of these should work
for kType in (ipDiffim.KernelCandidateF.PCA,):
for kMethod in (kc.getKernelSolution,
kc.getKernel,
kc.getBackground,
kc.getKsum,
kc.getKernelImage,
kc.getImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception:
pass
else:
print(kMethod)
self.fail()
self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT))
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "DF"
self.subconfig = self.config.kernel.active
self.policy = pexConfig.makePolicy(self.subconfig)
self.policy.set("useRegularization", False)
self.policy.set("checkConditionNumber", False) # I am making shady kernels by hand
self.policy.set("useCoreStats", False) # I am making off-center resids
self.kList = ipDiffim.makeKernelBasisList(self.subconfig)
self.size = 51
def testGaussian(self, imsize = 50):
# Convolve a delta function with a known gaussian; try to
# recover using delta-function basis
gsize = self.policy.getInt("kernelSize")
tsize = imsize + gsize
gaussFunction = afwMath.GaussianFunction2D(2, 3)
gaussKernel = afwMath.AnalyticKernel(gsize, gsize, gaussFunction)
kImageIn = afwImage.ImageD(afwGeom.Extent2I(gsize, gsize))
gaussKernel.computeImage(kImageIn, False)
# template image with a single hot pixel in the exact center
tmi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize))
tmi.set(0, 0x0, 1e-4)
cpix = tsize // 2
tmi.set(cpix, cpix, (1, 0x0, 1))
# science image
smi = afwImage.MaskedImageF(tmi.getDimensions())
afwMath.convolve(smi, tmi, gaussKernel, False)
# get the actual kernel sum (since the image is not infinite)
gscaling = afwMath.makeStatistics(smi, afwMath.SUM).getValue(afwMath.SUM)
# grab only the non-masked subregion
bbox = gaussKernel.shrinkBBox(smi.getBBox(afwImage.LOCAL))
tmi2 = afwImage.MaskedImageF(tmi, bbox, afwImage.LOCAL)
smi2 = afwImage.MaskedImageF(smi, bbox, afwImage.LOCAL)
# make sure its a valid subregion!
for j in range(tmi2.getHeight()):
for i in range(tmi2.getWidth()):
self.assertEqual(tmi2.getMask().get(i, j), 0)
self.assertEqual(smi2.getMask().get(i, j), 0)
kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi2, smi2, self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
kImageOut = kc.getImage()
soln = kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT)
self.assertAlmostEqual(soln.getKsum(), gscaling)
self.assertAlmostEqual(soln.getBackground(), 0.0)
for j in range(kImageOut.getHeight()):
for i in range(kImageOut.getWidth()):
self.assertAlmostEqual(kImageOut.get(i, j)/kImageIn.get(i, j), 1.0, 5)
def testSourceStats(self):
source = self.ss.addNew()
source.setId(1)
source.set(self.table.getCentroidKey().getX(), 276)
source.set(self.table.getCentroidKey().getY(), 717)
source.set("slot_PsfFlux_instFlux", 1.)
kc = ipDiffim.KernelCandidateF(source,
self.templateExposure2.getMaskedImage(),
self.scienceImage2.getMaskedImage(),
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3):
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True)
sIm *= scaling
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm,
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT),
kSum=scaling)
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), deep=True)
sIm += bg
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm,
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.verifyDeltaFunctionSolution(kc.getKernelSolution(ipDiffim.KernelCandidateF.RECENT),
bg=bg)
def testSourceStats(self):
source = self.ss.addNew()
source.setId(1)
source.set(self.table.getCentroidSlot().getMeasKey().getX(), 276)
source.set(self.table.getCentroidSlot().getMeasKey().getY(), 717)
source.set("slot_PsfFlux_instFlux", 1.)
kc = ipDiffim.KernelCandidateF(source,
self.templateExposure2.getMaskedImage(),
self.scienceImage2.getMaskedImage(),
self.ps)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def runAlSpatialModel(self, sko, bgo):
basisList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy.set('spatialKernelOrder', sko)
self.policy.set('spatialBgOrder', bgo)
self.policy.set('fitForBackground', True)
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(self.size * 10, self.size * 10))
bsikv = ipDiffim.BuildSingleKernelVisitorF(basisList, self.policy)
bspkv = ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox,
self.policy)
for x in range(1, self.size, 10):
for y in range(1, self.size, 10):
cand = self.makeCandidate(1.0, x, y)
bsikv.processCandidate(cand)
bspkv.processCandidate(cand)
bspkv.solveLinearEquation()
sk, sb = bspkv.getSolutionPair()
# Kernel
if sko == 0:
# Specialization for speedup
spatialKernelSolution = sk.getKernelParameters()
# One term for each basis function
self.assertEqual(len(spatialKernelSolution), len(basisList))
else:
spatialKernelSolution = sk.getSpatialParameters()
nSpatialTerms = int(0.5 * (sko + 1) * (sko + 2))
# One model for each basis function
self.assertEqual(len(spatialKernelSolution), len(basisList))
# First basis has no spatial variation
for i in range(1, nSpatialTerms):
self.assertEqual(spatialKernelSolution[0][i], 0.)
# All bases have correct number of terms
for i in range(len(spatialKernelSolution)):
self.assertEqual(len(spatialKernelSolution[i]), nSpatialTerms)
# Background
spatialBgSolution = sb.getParameters()
nBgTerms = int(0.5 * (bgo + 1) * (bgo + 2))
self.assertEqual(len(spatialBgSolution), nBgTerms)
def testModelType(self):
bbox = geom.Box2I(geom.Point2I(10, 10), geom.Extent2I(10, 10))
basisList = ipDiffim.makeKernelBasisList(self.subconfig)
self.ps["spatialModelType"] = "polynomial"
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.ps)
self.ps["spatialModelType"] = "chebyshev1"
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.ps)
try:
self.ps["spatialModelType"] = "foo"
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.ps)
except Exception:
pass
else:
self.fail()
def testModelType(self):
bbox = afwGeom.Box2I(afwGeom.Point2I(10, 10), afwGeom.Extent2I(10, 10))
basisList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy.set("spatialModelType", "polynomial")
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
self.policy.set("spatialModelType", "chebyshev1")
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
try:
self.policy.set("spatialModelType", "foo")
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
except Exception:
pass
else:
self.fail()
def runAlSpatialModel(self, sko, bgo):
basisList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy.set('spatialKernelOrder', sko)
self.policy.set('spatialBgOrder', bgo)
self.policy.set('fitForBackground', True)
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(self.size*10, self.size*10))
bsikv = ipDiffim.BuildSingleKernelVisitorF(basisList, self.policy)
bspkv = ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
for x in range(1, self.size, 10):
for y in range(1, self.size, 10):
cand = self.makeCandidate(1.0, x, y)
bsikv.processCandidate(cand)
bspkv.processCandidate(cand)
bspkv.solveLinearEquation()
sk, sb = bspkv.getSolutionPair()
# Kernel
if sko == 0:
# Specialization for speedup
spatialKernelSolution = sk.getKernelParameters()
# One term for each basis function
self.assertEqual(len(spatialKernelSolution), len(basisList))
else:
spatialKernelSolution = sk.getSpatialParameters()
nSpatialTerms = int(0.5 * (sko + 1) * (sko + 2))
# One model for each basis function
self.assertEqual(len(spatialKernelSolution), len(basisList))
# First basis has no spatial variation
for i in range(1, nSpatialTerms):
self.assertEqual(spatialKernelSolution[0][i], 0.)
# All bases have correct number of terms
for i in range(len(spatialKernelSolution)):
self.assertEqual(len(spatialKernelSolution[i]), nSpatialTerms)
# Background
spatialBgSolution = sb.getParameters()
nBgTerms = int(0.5 * (bgo + 1) * (bgo + 2))
self.assertEqual(len(spatialBgSolution), nBgTerms)
def testModelType(self):
bbox = afwGeom.Box2I(afwGeom.Point2I(10, 10),
afwGeom.Extent2I(10, 10))
basisList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy.set("spatialModelType", "polynomial")
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
self.policy.set("spatialModelType", "chebyshev1")
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
try:
self.policy.set("spatialModelType", "foo")
ipDiffim.BuildSpatialKernelVisitorF(basisList, bbox, self.policy)
except Exception:
pass
else:
self.fail()
def testSourceStats(self):
# Original and uninitialized
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
source = self.ss.addNew()
source.setId(1)
source.set(self.table.getCentroidKey().getX(), 276)
source.set(self.table.getCentroidKey().getY(), 717)
source.set(self.table.getPsfFluxKey(), 1.)
kc = ipDiffim.KernelCandidateF(source,
self.templateExposure2.getMaskedImage(),
self.scienceImage2.getMaskedImage(),
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def testSourceStats(self):
# Original and uninitialized
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
source = self.ss.addNew()
source.setId(1)
source.set(self.table.getCentroidKey().getX(), 276)
source.set(self.table.getCentroidKey().getY(), 717)
source.set(self.table.getPsfFluxKey(), 1.)
kc = ipDiffim.KernelCandidateF(source,
self.templateExposure2.getMaskedImage(),
self.scienceImage2.getMaskedImage(),
self.policy)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def testZeroVariance(self, imsize=50):
gsize = self.ps["kernelSize"]
tsize = imsize + gsize
tmi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize))
tmi.set(0, 0x0, 1.0)
cpix = tsize // 2
tmi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 0.0)
smi = afwImage.MaskedImageF(geom.Extent2I(tsize, tsize))
smi.set(0, 0x0, 1.0)
smi[cpix, cpix, afwImage.LOCAL] = (1, 0x0, 0.0)
kList = ipDiffim.makeKernelBasisList(self.subconfig)
self.ps["constantVarianceWeighting"] = False
kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.ps)
try:
kc.build(kList)
except Exception:
pass
else:
self.fail()
def testZeroVariance(self, imsize=50):
gsize = self.policy.getInt("kernelSize")
tsize = imsize + gsize
tmi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize))
tmi.set(0, 0x0, 1.0)
cpix = tsize // 2
tmi.set(cpix, cpix, (1, 0x0, 0.0))
smi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize))
smi.set(0, 0x0, 1.0)
smi.set(cpix, cpix, (1, 0x0, 0.0))
kList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy.set("constantVarianceWeighting", False)
kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.policy)
try:
kc.build(kList)
except Exception:
pass
else:
self.fail()
def testZeroVariance(self, imsize = 50):
gsize = self.policy.getInt("kernelSize")
tsize = imsize + gsize
tmi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize))
tmi.set(0, 0x0, 1.0)
cpix = tsize // 2
tmi.set(cpix, cpix, (1, 0x0, 0.0))
smi = afwImage.MaskedImageF(afwGeom.Extent2I(tsize, tsize))
smi.set(0, 0x0, 1.0)
smi.set(cpix, cpix, (1, 0x0, 0.0))
kList = ipDiffim.makeKernelBasisList(self.subconfig)
self.policy.set("constantVarianceWeighting", False)
kc = ipDiffim.KernelCandidateF(0.0, 0.0, tmi, smi, self.policy)
try:
kc.build(kList)
except Exception:
pass
else:
self.fail()
def testSubtractMaskedImages(self):
# Lets do some additional testing here to make sure we recover
# the known spatial model. No background, just the faked
# alard-lupton basis set. The rest of matchMaskedImages() and
# subtractMaskedImages() functionality is tested by the
# Exposure-based methods.
fakeCoeffs = diffimTools.fakeCoeffs()
# Quick note; you shouldn't change confake here, since the
# candidates in the KernelCellSet are initialized in
# makeFakeKernelSet
tMi, sMi, sK, kcs, confake = diffimTools.makeFakeKernelSet(
bgValue=0.0, addNoise=False)
svar = sMi.getVariance()
svar.set(1.0)
tvar = tMi.getVariance()
tvar.set(1.0)
basisList = ipDiffim.makeKernelBasisList(confake.kernel.active)
psfMatchAL = ipDiffim.ImagePsfMatchTask(config=confake)
spatialSolution, psfMatchingKernel, backgroundModel = psfMatchAL._solve(
kcs, basisList)
fitCoeffs = psfMatchingKernel.getSpatialParameters()
for b in range(len(fakeCoeffs)):
for s in range(len(fakeCoeffs[b])):
if fakeCoeffs[b][s] == 0.0:
self.assertAlmostEqual(fitCoeffs[b][s], 0.0)
else:
# OUTSTANDING ISSUE - WHY IS THIS ONE TERM OFF!?!?
if b != 4 and s != 0:
self.assertAlmostEqual(
fitCoeffs[b][s] / fakeCoeffs[b][s], 1.0, 1)
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "AL"
self.subconfig = self.config.kernel.active
# Test was put together before the min size went to 21
self.subconfig.kernelSize = 19
self.subconfig.scaleByFwhm = False
self.subconfig.fitForBackground = True
self.subconfig.spatialModelType = "polynomial"
self.policy = pexConfig.makePolicy(self.subconfig)
self.smi = afwImage.MaskedImageF('tests/compareToHotpants/scienceMI.fits')
self.tmi = afwImage.MaskedImageF('tests/compareToHotpants/templateMI.fits')
self.smi.setXY0(0, 0)
self.tmi.setXY0(0, 0)
# Run detection
# detConfig = self.subconfig.detectionConfig
# Note here regarding detConfig:
#
# If I set detThresholdType = "pixel_stdev", I get slightly
# different centroids than if I use "stdev". These different
# centroids screw up the testing since hotpants was hardcoded to
# use the "stdev" centroids. For completeness these are:
#
# 32 32
# 96 32
# 160 32
# 96 95
# 31 96
# 160 96
# 96 160
# 160 160
# 32 160
# As of Winter2013, KernelCandidateDetectionF does not return
# these exact centroids anymore, so I need to hardcode them
# in.
self.footprints = []
for xc, yc in [(32, 32), (96, 32), (160, 32),
(96, 95), (31, 96), (160, 96),
(96, 160), (160, 160), (32, 160)]:
self.footprints.append(afwDet.Footprint(afwGeom.SpanSet(
afwGeom.Box2I(afwGeom.Point2I(xc, yc),
afwGeom.Extent2I(1, 1)))))
# Make a basis list that hotpants has been run with
nGauss = 1
sGauss = [3.]
dGauss = [3]
self.subconfig.alardNGauss = nGauss
self.subconfig.alardSigGauss = sGauss
self.subconfig.alardDegGauss = dGauss
basisList0 = ipDiffim.makeKernelBasisList(self.subconfig)
# HP does things in a different order, and with different normalization, so reorder list
order = [0, 2, 5, 9, 1, 4, 8, 3, 7, 6]
scaling = [1.000000e+00,
8.866037e-02,
1.218095e+01,
5.099318e-03,
8.866037e-02,
4.179772e-02,
1.138120e-02,
1.218095e+01,
1.138120e-02,
5.099318e-03]
self.basisList = []
for i in range(len(order)):
im = afwImage.ImageD(basisList0[order[i]].getDimensions())
basisList0[order[i]].computeImage(im, False)
im /= scaling[i]
# im.writeFits('k%d.fits' % (i))
k = afwMath.FixedKernel(im)
self.basisList.append(k)
# And a place to put candidates
self.kernelCellSet = afwMath.SpatialCellSet(afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(self.smi.getWidth(),
self.smi.getHeight())),
self.policy.getInt("sizeCellX"),
self.policy.getInt("sizeCellY"))
# There are some -1 factors that come from differences in how
# convolution is done. Some resulting convovled images end up
# being a factor of -1 different, therefore the coefficients
# need to be a factor of -1 different as well.
self.parity = [1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1]
def setUp(self):
self.configAL = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.configAL.kernel.name = "AL"
self.subconfigAL = self.configAL.kernel.active
self.configDF = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.configDF.kernel.name = "DF"
self.subconfigDF = self.configDF.kernel.active
self.configDFr = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.configDFr.kernel.name = "DF"
self.subconfigDFr = self.configDFr.kernel.active
self.subconfigDF.useRegularization = False
self.subconfigDFr.useRegularization = True
self.subconfigDFr.lambdaValue = 1000.0
self.subconfigAL.fitForBackground = fitForBackground
self.subconfigDF.fitForBackground = fitForBackground
self.subconfigDFr.fitForBackground = fitForBackground
self.subconfigAL.constantVarianceWeighting = constantVarianceWeighting
self.subconfigDF.constantVarianceWeighting = constantVarianceWeighting
self.subconfigDFr.constantVarianceWeighting = constantVarianceWeighting
self.kListAL = ipDiffim.makeKernelBasisList(self.subconfigAL)
self.kListDF = ipDiffim.makeKernelBasisList(self.subconfigDF)
self.kListDFr = ipDiffim.makeKernelBasisList(self.subconfigDFr)
self.hMatDFr = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfigDFr))
self.bskvAL = ipDiffim.BuildSingleKernelVisitorF(self.kListAL, pexConfig.makePolicy(self.subconfigAL))
self.bskvDF = ipDiffim.BuildSingleKernelVisitorF(self.kListDF, pexConfig.makePolicy(self.subconfigDF))
self.bskvDFr = ipDiffim.BuildSingleKernelVisitorF(self.kListDFr,
pexConfig.makePolicy(self.subconfigDF),
self.hMatDFr)
defSciencePath = globals()['defSciencePath']
defTemplatePath = globals()['defTemplatePath']
if defSciencePath and defTemplatePath:
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
else:
defDataDir = lsst.utils.getPackageDir('afwdata')
defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0",
"v26-e0-c011-a00.sci.fits")
defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0",
"v5-e0-c011-a00.sci.fits")
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
warper = afwMath.Warper.fromConfig(self.subconfigAL.warpingConfig)
self.templateExposure = warper.warpExposure(self.scienceExposure.getWcs(), self.templateExposure,
destBBox=self.scienceExposure.getBBox())
#
tmi = self.templateExposure.getMaskedImage()
smi = self.scienceExposure.getMaskedImage()
# Object detection
detConfig = self.subconfigAL.detectionConfig
detPolicy = pexConfig.makePolicy(detConfig)
detPolicy.set("detThreshold", 50.)
detPolicy.set("detThresholdType", "stdev")
detPolicy.set("detOnTemplate", False)
kcDetect = ipDiffim.KernelCandidateDetectionF(detPolicy)
kcDetect.apply(tmi, smi)
self.footprints = kcDetect.getFootprints()
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
print templateIdList
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma = scienceSigmaPost,
doSmooth = not self.doPreConvolve,
idFactory = idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
False)
else:
raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
"but template sources not available. Cannot match science " +
"sources with template sources. Run process* on data from " +
"which templates are built.")
kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [k for i,k in enumerate(kernelSources) if i % self.config.controlStepSize]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory
)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
exposure.getWcs(), exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId() : x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey))) for m in wcsResults.matches]
colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
+ 2.5*numpy.log10(srcToMatch[x].get("r"))
for x in sids if x in srcToMatch.keys()])
dlong = numpy.array([r[0].asArcseconds() for s,r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
dlat = numpy.array([r[1].asArcseconds() for s,r in zip(sids, cresiduals)
if s in srcToMatch.keys()])
idx1 = numpy.where(colors<self.sourceSelector.config.grMin)
idx2 = numpy.where((colors>=self.sourceSelector.config.grMin)&
(colors<=self.sourceSelector.config.grMax))
idx3 = numpy.where(colors>self.sourceSelector.config.grMax)
rms1Long = IqrToSigma*(numpy.percentile(dlong[idx1],75)-numpy.percentile(dlong[idx1],25))
rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1],75)-numpy.percentile(dlat[idx1],25))
rms2Long = IqrToSigma*(numpy.percentile(dlong[idx2],75)-numpy.percentile(dlong[idx2],25))
rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2],75)-numpy.percentile(dlat[idx2],25))
rms3Long = IqrToSigma*(numpy.percentile(dlong[idx3],75)-numpy.percentile(dlong[idx3],25))
rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3],75)-numpy.percentile(dlat[idx3],25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
rms1Long,
numpy.median(dlat[idx1]),
rms1Lat))
self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
rms2Long,
numpy.median(dlat[idx2]),
rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
rms3Long,
numpy.median(dlat[idx3]),
rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Running diaSource detection")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
self.measurement.run(diaSources, subtractedExposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = measAstrom.AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = measAstrom.AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def run(self, sensorRef):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = long(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
if self.config.useWinter2013Hacks and self.config.winter2013borderMask > 0:
self.log.warn("USING WINTER2013 HACK: MASKING BORDER PIXELS")
bbox = exposure.getBBox(afwImage.PARENT)
bbox.grow(-self.config.winter2013borderMask)
self.setEdgeBits(exposure.getMaskedImage(), bbox,
exposure.getMaskedImage().getMask().getPlaneBitMask("NO_DATA"))
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
ctr = afwGeom.Box2D(exposure.getBBox(afwImage.PARENT)).getCenter()
psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
# sigma of PSF of template image before warping
ctr = afwGeom.Box2D(templateExposure.getBBox(afwImage.PARENT)).getCenter()
psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getKernel().getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = psf
afwMath.convolve(destMI, srcMI, preConvPsf.getKernel(), convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn("Src product does not exist; running detection, measurement, selection")
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma = scienceSigmaPost,
doSmooth = not self.doPreConvolve,
idFactory = idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
referenceFwhmPix = scienceSigmaPost * FwhmPerSigma,
targetFwhmPix = templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
self.kcQa = diUtils.KernelCandidateQa(nparam, self.log)
selectSources = self.kcQa.addToSchema(selectSources)
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig())
astromRet = astrometer.useKnownWcs(selectSources, exposure=exposure)
matches = astromRet.matches
kernelSources = self.sourceSelector.selectSources(exposure, selectSources, matches=matches)
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
[kernelSources.remove(x) for x in controlSources]
self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)" \
% (len(kernelSources), len(selectSources), len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# First step: we need to subtract the background out
# for detection and measurement. Use large binsize
# for the background estimation.
binsize = self.config.templateBgBinSize
# Second step: we need to run detection on the
# background-subtracted template
#
# Estimate FWHM for detection
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma = templateSigma,
doSmooth = True,
idFactory = idFactory,
binsize = binsize,
)
# Third step: we need to fit the relative astrometry.
#
# One problem is that the SIP fits are w.r.t. CRPIX,
# and these coadd patches have the CRPIX of the entire
# tract, i.e. off the image. This causes
# register.fitWcs to fail. A workaround for now is to
# re-fit the Wcs which returns with a CRPIX that is on
# the image, and *then* to fit for the relative Wcs.
#
# Requires low Sip order to avoid overfitting
# useWinter2013Hacks includes us using the deep calexp
# as the template. In this case we don't need to
# refit the Wcs.
if not self.config.useWinter2013Hacks:
sipOrder = self.config.templateSipOrder
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(sipOrder=sipOrder))
newWcs = astrometer.determineWcs(templateSources, templateExposure).getWcs()
results = self.register.run(templateSources, newWcs,
templateExposure.getBBox(afwImage.PARENT), selectSources)
else:
if self.config.winter2013WcsShift > 0.0:
offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
self.config.winter2013WcsShift)
cKey = templateSources[0].getTable().getCentroidKey()
for source in templateSources:
centroid = source.get(cKey)
source.set(cKey, centroid+offset)
elif self.config.winter2013WcsRms > 0.0:
cKey = templateSources[0].getTable().getCentroidKey()
for source in templateSources:
offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
self.config.winter2013WcsRms*numpy.random.normal())
centroid = source.get(cKey)
source.set(cKey, centroid+offset)
results = self.register.run(templateSources, templateExposure.getWcs(),
templateExposure.getBBox(afwImage.PARENT), selectSources)
warpedExp = self.register.warpExposure(templateExposure, results.wcs,
exposure.getWcs(), exposure.getBBox(afwImage.PARENT))
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
refCoordKey = results.matches[0].first.getTable().getCoordKey()
inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
sids = [m.first.getId() for m in results.matches]
positions = [m.first.get(refCoordKey) for m in results.matches]
residuals = [m.first.get(refCoordKey).getOffsetFrom(
results.wcs.pixelToSky(m.second.get(inCentroidKey))) for
m in results.matches]
allresids = dict(zip(sids, zip(positions, residuals)))
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
#Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure = templateExposure,
scienceExposure = exposure,
scienceFwhmPix = scienceSigmaPost * FwhmPerSigma,
templateFwhmPix = templateSigma * FwhmPerSigma,
candidateList = kernelSources,
convolveTemplate = self.config.convolveTemplate,
doWarping = not self.config.doUseRegister
)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Running diaSource detection")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
templateExposure, templateSources = self.getTemplate(exposure, sensorRef)
subtractedExposure.setPsf(templateExposure.getPsf())
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table = table,
exposure = subtractedExposure,
doSmooth = not self.config.doPreConvolve
)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" % (len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if len(diaSources) < self.config.maxDiaSourcesToMeasure:
self.dipolemeasurement.run(subtractedExposure, diaSources)
else:
self.measurement.run(subtractedExposure, diaSources)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
# This does not do what I expect so I cobbled together a brute force method in python
srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel, True)
srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for \
srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
len(diaSources)))
else:
self.log.warn("Src product does not exist; cannot match with diaSources")
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
astrometer = measAstrom.Astrometry(measAstrom.MeasAstromConfig(catalogMatchDist=matchRadAsec))
astromRet = astrometer.useKnownWcs(diaSources, exposure=exposure)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn("No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for \
refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if srcMatchDict.has_key(sid):
diaSource.set("srcMatchId", srcMatchDict[sid])
if refMatchDict.has_key(sid):
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sourceWriteFlags = (0 if self.config.doWriteHeavyFootprintsInSources
else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc", flags=sourceWriteFlags)
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cast_KernelCandidateF(cand))
# Get basis list to build control sample kernels
basisList = afwMath.cast_LinearCombinationKernel(
kernelCandList[0].getKernel(KernelCandidateF.ORIG)).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandList(controlSources, subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, dobuild=True, basisList=basisList)
self.kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
dof=nparam)
self.kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
if self.config.doDetection:
self.kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
else:
self.kcQa.aggregate(selectSources, self.metadata, allresids)
#Persist using butler
sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
return pipeBase.Struct(
subtractedExposure = subtractedExposure,
subtractRes = subtractRes,
sources = diaSources,
)
def setUp(self):
self.configAL = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.configAL.kernel.name = "AL"
self.subconfigAL = self.configAL.kernel.active
self.configDF = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.configDF.kernel.name = "DF"
self.subconfigDF = self.configDF.kernel.active
self.configDFr = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.configDFr.kernel.name = "DF"
self.subconfigDFr = self.configDFr.kernel.active
self.subconfigDF.useRegularization = False
self.subconfigDFr.useRegularization = True
self.subconfigDFr.lambdaValue = 1000.0
self.subconfigAL.fitForBackground = fitForBackground
self.subconfigDF.fitForBackground = fitForBackground
self.subconfigDFr.fitForBackground = fitForBackground
self.subconfigAL.constantVarianceWeighting = constantVarianceWeighting
self.subconfigDF.constantVarianceWeighting = constantVarianceWeighting
self.subconfigDFr.constantVarianceWeighting = constantVarianceWeighting
self.kListAL = ipDiffim.makeKernelBasisList(self.subconfigAL)
self.kListDF = ipDiffim.makeKernelBasisList(self.subconfigDF)
self.kListDFr = ipDiffim.makeKernelBasisList(self.subconfigDFr)
self.hMatDFr = ipDiffim.makeRegularizationMatrix(
pexConfig.makePropertySet(self.subconfigDFr))
self.bskvAL = ipDiffim.BuildSingleKernelVisitorF(
self.kListAL, pexConfig.makePropertySet(self.subconfigAL))
self.bskvDF = ipDiffim.BuildSingleKernelVisitorF(
self.kListDF, pexConfig.makePropertySet(self.subconfigDF))
self.bskvDFr = ipDiffim.BuildSingleKernelVisitorF(
self.kListDFr, pexConfig.makePropertySet(self.subconfigDF),
self.hMatDFr)
defSciencePath = globals()['defSciencePath']
defTemplatePath = globals()['defTemplatePath']
if defSciencePath and defTemplatePath:
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
else:
defDataDir = lsst.utils.getPackageDir('afwdata')
defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci",
"v26-e0", "v26-e0-c011-a00.sci.fits")
defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci",
"v5-e0", "v5-e0-c011-a00.sci.fits")
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
warper = afwMath.Warper.fromConfig(self.subconfigAL.warpingConfig)
self.templateExposure = warper.warpExposure(
self.scienceExposure.getWcs(),
self.templateExposure,
destBBox=self.scienceExposure.getBBox())
#
tmi = self.templateExposure.getMaskedImage()
smi = self.scienceExposure.getMaskedImage()
# Object detection
detConfig = self.subconfigAL.detectionConfig
detps = pexConfig.makePropertySet(detConfig)
detps["detThreshold"] = 50.
detps["detThresholdType"] = "stdev"
detps["detOnTemplate"] = False
kcDetect = ipDiffim.KernelCandidateDetectionF(detps)
kcDetect.apply(tmi, smi)
self.footprints = kcDetect.getFootprints()
def testMakeKernelBasisList(self):
ks = ipDiffim.makeKernelBasisList(self.subconfigAL)
self.alardLuptonTest(ks)
ks = ipDiffim.makeKernelBasisList(self.subconfigDF)
self.deltaFunctionTest(ks)
kc.getBackground, kc.getKsum, kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception, e:
pass
else:
self.fail()
try:
kc.getImage()
except Exception:
pass
else:
self.fail()
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def testDeltaFunctionScaled(self, scaling=2.7, bg=11.3):
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(),
True)
sIm *= scaling
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm, self.policy)
kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception, e:
pass
else:
self.fail()
try:
kc.getImage()
except Exception:
pass
else:
self.fail()
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def testDeltaFunctionScaled(self, scaling = 2.7, bg = 11.3):
if not self.defDataDir:
print >> sys.stderr, "Warning: afwdata is not set up"
return
sIm = afwImage.MaskedImageF(self.templateExposure2.getMaskedImage(), True)
sIm *= scaling
kc = ipDiffim.KernelCandidateF(self.x02, self.y02,
self.templateExposure2.getMaskedImage(),
sIm,
self.policy)
def testSourceConstructor(self):
source = self.ss.addNew()
source.setId(1)
source.set(self.table.getCentroidSlot().getMeasKey().getX(), 276)
source.set(self.table.getCentroidSlot().getMeasKey().getY(), 717)
source.set("slot_PsfFlux_instFlux", 1.)
kc = ipDiffim.KernelCandidateF(source,
self.templateExposure2.getMaskedImage(),
self.scienceImage2.getMaskedImage(),
self.ps)
# Kernel not initialized
self.assertEqual(kc.isInitialized(), False)
# Check that the source is set
self.assertEqual(kc.getSource(), source)
self.assertEqual(kc.getCandidateRating(), source.getPsfInstFlux())
# But this should be set on construction
try:
kc.getCandidateRating()
except Exception as e:
print(e)
self.fail()
# And these should be filled
try:
kc.getTemplateMaskedImage()
kc.getScienceMaskedImage()
except Exception as e:
print(e)
self.fail()
# And of the right type
self.assertEqual(type(kc.getTemplateMaskedImage()),
afwImage.MaskedImageF)
self.assertEqual(type(kc.getScienceMaskedImage()),
afwImage.MaskedImageF)
# None of these should work
for kType in (ipDiffim.KernelCandidateF.ORIG,
ipDiffim.KernelCandidateF.PCA,
ipDiffim.KernelCandidateF.RECENT):
for kMethod in (kc.getKernelSolution, kc.getKernel,
kc.getBackground, kc.getKsum, kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception:
pass
else:
self.fail()
try:
kc.getImage()
except Exception:
pass
else:
self.fail()
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = int(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
template = self.getTemplate.run(exposure,
sensorRef,
templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
# sigma of PSF of template image before warping
templateSigma = templateExposure.getPsf().computeShape(
).getDeterminantRadius()
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel(
).getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight,
scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(),
convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn(
"Src product does not exist; running detection, measurement, selection"
)
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma=scienceSigmaPost,
doSmooth=not self.doPreConvolve,
idFactory=idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(
makeKernelBasisList(
self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(
exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
mc = afwTable.MatchControl()
mc.findOnlyClosest = False
matches = afwTable.matchRaDec(templateSources,
selectSources,
1.0 * afwGeom.arcseconds, mc)
else:
raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
+
"but template sources not available. Cannot match science "
+
"sources with template sources. Run process* on data from "
+ "which templates are built.")
kernelSources = self.sourceSelector.selectStars(
exposure, selectSources, matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [
k for i, k in enumerate(kernelSources)
if i % self.config.controlStepSize
]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=self.sourceSelector.config.grMax,
grMax=99.999))
redSources = redSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=-99.999,
grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources),
len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources,
templateExposure,
selectSources)
warpedExp = self.register.warpExposure(templateExposure,
wcsResults.wcs,
exposure.getWcs(),
exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId(): x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable(
).getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable(
).getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [
m.first.get(refCoordKey) for m in wcsResults.matches
]
residuals = [
m.first.get(refCoordKey).getOffsetFrom(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [
m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
colors = numpy.array([
-2.5 * numpy.log10(srcToMatch[x].get("g")) +
2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids
if x in srcToMatch.keys()
])
dlong = numpy.array([
r[0].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
dlat = numpy.array([
r[1].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
idx1 = numpy.where(
colors < self.sourceSelector.config.grMin)
idx2 = numpy.where(
(colors >= self.sourceSelector.config.grMin)
& (colors <= self.sourceSelector.config.grMax))
idx3 = numpy.where(
colors > self.sourceSelector.config.grMax)
rms1Long = IqrToSigma * \
(numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) -
numpy.percentile(dlat[idx1], 25))
rms2Long = IqrToSigma * \
(numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) -
numpy.percentile(dlat[idx2], 25))
rms3Long = IqrToSigma * \
(numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) -
numpy.percentile(dlat[idx3], 25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx1]), rms1Long,
numpy.median(dlat[idx1]), rms1Lat))
self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx2]), rms2Long,
numpy.median(dlat[idx2]), rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx3]), rms3Long,
numpy.median(dlat[idx3]), rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian",
numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian",
numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian",
numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian",
numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian",
numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian",
numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
# Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure,
self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Computing diffim PSF")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(
exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
# already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
if self.config.doDecorrelation and self.config.doSubtract:
decorrResult = self.decorrelate.run(exposure, templateExposure,
subtractedExposure,
subtractRes.psfMatchingKernel)
subtractedExposure = decorrResult.correctedExposure
if self.config.doDetection:
self.log.info("Running diaSource detection")
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" %
(len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if not self.config.doDipoleFitting:
self.measurement.run(diaSources, subtractedExposure)
else:
if self.config.doSubtract:
self.measurement.run(diaSources, subtractedExposure,
exposure,
subtractRes.matchedExposure)
else:
self.measurement.run(diaSources, subtractedExposure,
exposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs(
).pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"),
diaSources, matchRadPixel)
srcMatchDict = dict([(srcMatch.second.getId(),
srcMatch.first.getId())
for srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" %
(len(srcMatchDict), len(diaSources)))
else:
self.log.warn(
"Src product does not exist; cannot match with diaSources"
)
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure,
sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn(
"No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info(
"Matched %d / %d diaSources to reference catalog" %
(len(refMatches), len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(),
refMatch.first.getId())
for refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if sid in srcMatchDict:
diaSource.set("srcMatchId", srcMatchDict[sid])
if sid in refMatchDict:
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources,
self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cand)
# Get basis list to build control sample kernels
basisList = kernelCandList[0].getKernel(
KernelCandidateF.ORIG).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel,
subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids,
diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources,
self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources,
diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def setUp(self):
self.config = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config.kernel.name = "AL"
self.subconfig = self.config.kernel.active
# Test was put together before the min size went to 21
self.subconfig.kernelSize = 19
self.subconfig.scaleByFwhm = False
self.subconfig.fitForBackground = True
self.subconfig.spatialModelType = "polynomial"
self.ps = pexConfig.makePropertySet(self.subconfig)
self.smi = afwImage.MaskedImageF(
'tests/compareToHotpants/scienceMI.fits')
self.tmi = afwImage.MaskedImageF(
'tests/compareToHotpants/templateMI.fits')
self.smi.setXY0(0, 0)
self.tmi.setXY0(0, 0)
# Run detection
# detConfig = self.subconfig.detectionConfig
# Note here regarding detConfig:
#
# If I set detThresholdType = "pixel_stdev", I get slightly
# different centroids than if I use "stdev". These different
# centroids screw up the testing since hotpants was hardcoded to
# use the "stdev" centroids. For completeness these are:
#
# 32 32
# 96 32
# 160 32
# 96 95
# 31 96
# 160 96
# 96 160
# 160 160
# 32 160
# As of Winter2013, KernelCandidateDetectionF does not return
# these exact centroids anymore, so I need to hardcode them
# in.
self.footprints = []
for xc, yc in [(32, 32), (96, 32), (160, 32), (96, 95), (31, 96),
(160, 96), (96, 160), (160, 160), (32, 160)]:
self.footprints.append(
afwDet.Footprint(
afwGeom.SpanSet(
geom.Box2I(geom.Point2I(xc, yc), geom.Extent2I(1,
1)))))
# Make a basis list that hotpants has been run with
nGauss = 1
sGauss = [3.]
dGauss = [3]
self.subconfig.alardNGauss = nGauss
self.subconfig.alardSigGauss = sGauss
self.subconfig.alardDegGauss = dGauss
basisList0 = ipDiffim.makeKernelBasisList(self.subconfig)
# HP does things in a different order, and with different normalization, so reorder list
order = [0, 2, 5, 9, 1, 4, 8, 3, 7, 6]
scaling = [
1.000000e+00, 8.866037e-02, 1.218095e+01, 5.099318e-03,
8.866037e-02, 4.179772e-02, 1.138120e-02, 1.218095e+01,
1.138120e-02, 5.099318e-03
]
self.basisList = []
for i in range(len(order)):
im = afwImage.ImageD(basisList0[order[i]].getDimensions())
basisList0[order[i]].computeImage(im, False)
im /= scaling[i]
# im.writeFits('k%d.fits' % (i))
k = afwMath.FixedKernel(im)
self.basisList.append(k)
# And a place to put candidates
self.kernelCellSet = afwMath.SpatialCellSet(
geom.Box2I(
geom.Point2I(0, 0),
geom.Extent2I(self.smi.getWidth(), self.smi.getHeight())),
self.ps["sizeCellX"], self.ps["sizeCellY"])
# There are some -1 factors that come from differences in how
# convolution is done. Some resulting convovled images end up
# being a factor of -1 different, therefore the coefficients
# need to be a factor of -1 different as well.
self.parity = [1, -1, 1, -1, -1, 1, -1, 1, -1, -1, 1]
def setUp(self, CFHT=True):
lambdaValue = 1.0
self.config1 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config1.kernel.name = "DF"
self.subconfig1 = self.config1.kernel.active
self.config2 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config2.kernel.name = "DF"
self.subconfig2 = self.config2.kernel.active
self.config3 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config3.kernel.name = "DF"
self.subconfig3 = self.config3.kernel.active
self.config4 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config4.kernel.name = "DF"
self.subconfig4 = self.config4.kernel.active
self.subconfig1.useRegularization = False
self.subconfig2.useRegularization = True
self.subconfig2.lambdaType = "absolute"
self.subconfig2.lambdaValue = lambdaValue
self.subconfig2.regularizationType = "centralDifference"
self.subconfig2.centralRegularizationStencil = 5
self.subconfig3.useRegularization = True
self.subconfig3.lambdaType = "absolute"
self.subconfig3.lambdaValue = lambdaValue
self.subconfig3.regularizationType = "centralDifference"
self.subconfig3.centralRegularizationStencil = 9
self.subconfig4.useRegularization = True
self.subconfig4.lambdaType = "absolute"
self.subconfig4.lambdaValue = lambdaValue
self.subconfig4.regularizationType = "forwardDifference"
self.subconfig4.forwardRegularizationOrders = [1, 2]
self.kList1 = ipDiffim.makeKernelBasisList(self.subconfig1)
self.bskv1 = ipDiffim.BuildSingleKernelVisitorF(
self.kList1, pexConfig.makePropertySet(self.subconfig1))
self.kList2 = ipDiffim.makeKernelBasisList(self.subconfig2)
self.hMat2 = ipDiffim.makeRegularizationMatrix(
pexConfig.makePropertySet(self.subconfig2))
self.bskv2 = ipDiffim.BuildSingleKernelVisitorF(
self.kList2, pexConfig.makePropertySet(self.subconfig2),
self.hMat2)
self.kList3 = ipDiffim.makeKernelBasisList(self.subconfig3)
self.hMat3 = ipDiffim.makeRegularizationMatrix(
pexConfig.makePropertySet(self.subconfig3))
self.bskv3 = ipDiffim.BuildSingleKernelVisitorF(
self.kList3, pexConfig.makePropertySet(self.subconfig3),
self.hMat3)
self.kList4 = ipDiffim.makeKernelBasisList(self.subconfig4)
self.hMat4 = ipDiffim.makeRegularizationMatrix(
pexConfig.makePropertySet(self.subconfig4))
self.bskv4 = ipDiffim.BuildSingleKernelVisitorF(
self.kList4, pexConfig.makePropertySet(self.subconfig4),
self.hMat4)
# known input images
defDataDir = lsst.utils.getPackageDir('afwdata')
if CFHT:
defSciencePath = os.path.join(defDataDir, 'CFHT', 'D4',
CFHTTORUN + '.fits')
defTemplatePath = os.path.join(defDataDir, 'CFHT', 'D4',
CFHTTORUN + '_tmpl.fits')
# no need to remap
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
else:
defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci",
"v26-e0", "v26-e0-c011-a00.sci")
defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci",
"v5-e0", "v5-e0-c011-a00.sci")
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
warper = afwMath.Warper.fromConfig(self.subconfig1.warpingConfig)
self.templateExposure = warper.warpExposure(
self.scienceExposure.getWcs(),
self.templateExposure,
destBBox=self.scienceExposure.getBBox())
diffimTools.backgroundSubtract(self.subconfig1.afwBackgroundConfig, [
self.scienceExposure.getMaskedImage(),
self.templateExposure.getMaskedImage()
])
#
tmi = self.templateExposure.getMaskedImage()
smi = self.scienceExposure.getMaskedImage()
detConfig = self.subconfig1.detectionConfig
detps = pexConfig.makePropertySet(detConfig)
detps["detThreshold"] = 50.
detps["detOnTemplate"] = False
kcDetect = ipDiffim.KernelCandidateDetectionF(detps)
kcDetect.apply(tmi, smi)
self.footprints = kcDetect.getFootprints()
def testSourceConstructor(self):
source = self.ss.addNew()
source.setId(1)
source.set(self.table.getCentroidKey().getX(), 276)
source.set(self.table.getCentroidKey().getY(), 717)
source.set("slot_PsfFlux_instFlux", 1.)
kc = ipDiffim.KernelCandidateF(source,
self.templateExposure2.getMaskedImage(),
self.scienceImage2.getMaskedImage(),
self.policy)
# Kernel not initialized
self.assertEqual(kc.isInitialized(), False)
# Check that the source is set
self.assertEqual(kc.getSource(), source)
self.assertEqual(kc.getCandidateRating(), source.getPsfInstFlux())
# But this should be set on construction
try:
kc.getCandidateRating()
except Exception as e:
print(e)
self.fail()
# And these should be filled
try:
kc.getTemplateMaskedImage()
kc.getScienceMaskedImage()
except Exception as e:
print(e)
self.fail()
# And of the right type
self.assertEqual(type(kc.getTemplateMaskedImage()), afwImage.MaskedImageF)
self.assertEqual(type(kc.getScienceMaskedImage()), afwImage.MaskedImageF)
# None of these should work
for kType in (ipDiffim.KernelCandidateF.ORIG,
ipDiffim.KernelCandidateF.PCA,
ipDiffim.KernelCandidateF.RECENT):
for kMethod in (kc.getKernelSolution,
kc.getKernel,
kc.getBackground,
kc.getKsum,
kc.getKernelImage,
kc.getDifferenceImage):
try:
kMethod(kType)
except Exception:
pass
else:
self.fail()
try:
kc.getImage()
except Exception:
pass
else:
self.fail()
kList = ipDiffim.makeKernelBasisList(self.subconfig)
kc.build(kList)
self.assertEqual(kc.isInitialized(), True)
fnum = 1
for switch in ['A', 'B', 'C']:
if switch == 'A':
# AL
config = subconfigAL
elif switch == 'B':
# AL with ~320 bases
config = subconfigAL
config.alardDegGauss = (15, 10, 5)
elif switch == 'C':
config = subconfigDF
config.useRegularization = False
kList = ipDiffim.makeKernelBasisList(config)
policy = pexConfig.makePolicy(config)
bskv = ipDiffim.BuildSingleKernelVisitorF(kList, policy)
# TEST 1
tmi, smi = makeTest1(doAddNoise)
kc = ipDiffim.makeKernelCandidate(0.0, 0.0, tmi, smi, policy)
bskv.processCandidate(kc)
kernel = kc.getKernel(ipDiffim.KernelCandidateF.ORIG)
kimage = afwImage.ImageD(kernel.getDimensions())
kernel.computeImage(kimage, False)
diffim = kc.getDifferenceImage(ipDiffim.KernelCandidateF.ORIG)
afwDisplay.Display(frame=fnum).mtv(tmi)
fnum = 1
for switch in ['A', 'B', 'C']:
if switch == 'A':
# Default Alard Lupton
config = subconfigAL
elif switch == 'B':
# Add more AL bases (typically 4 3 2)
config = subconfigAL
config.alardDegGauss = (8, 6, 4)
elif switch == 'C':
# Delta function
config = subconfigDF
config.useRegularization = False
kList = ipDiffim.makeKernelBasisList(config)
policy = pexConfig.makePolicy(config)
bskv = ipDiffim.BuildSingleKernelVisitorF(kList, policy)
# TEST 1
tmi, smi = makeTest1(doAddNoise)
kc = ipDiffim.makeKernelCandidate(0.0, 0.0, tmi, smi, policy)
bskv.processCandidate(kc)
kernel = kc.getKernel(ipDiffim.KernelCandidateF.ORIG)
kimage = afwImage.ImageD(kernel.getDimensions())
kernel.computeImage(kimage, False)
diffim = kc.getDifferenceImage(ipDiffim.KernelCandidateF.ORIG)
ds9.mtv(tmi, frame=fnum)
def testMakeKernelBasisList(self):
ks = ipDiffim.makeKernelBasisList(self.subconfigAL)
self.alardLuptonTest(ks)
ks = ipDiffim.makeKernelBasisList(self.subconfigDF)
self.deltaFunctionTest(ks)
def setUp(self, CFHT=True):
lambdaValue = 1.0
self.config1 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config1.kernel.name = "DF"
self.subconfig1 = self.config1.kernel.active
self.config2 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config2.kernel.name = "DF"
self.subconfig2 = self.config2.kernel.active
self.config3 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config3.kernel.name = "DF"
self.subconfig3 = self.config3.kernel.active
self.config4 = ipDiffim.ImagePsfMatchTask.ConfigClass()
self.config4.kernel.name = "DF"
self.subconfig4 = self.config4.kernel.active
self.subconfig1.useRegularization = False
self.subconfig2.useRegularization = True
self.subconfig2.lambdaType = "absolute"
self.subconfig2.lambdaValue = lambdaValue
self.subconfig2.regularizationType = "centralDifference"
self.subconfig2.centralRegularizationStencil = 5
self.subconfig3.useRegularization = True
self.subconfig3.lambdaType = "absolute"
self.subconfig3.lambdaValue = lambdaValue
self.subconfig3.regularizationType = "centralDifference"
self.subconfig3.centralRegularizationStencil = 9
self.subconfig4.useRegularization = True
self.subconfig4.lambdaType = "absolute"
self.subconfig4.lambdaValue = lambdaValue
self.subconfig4.regularizationType = "forwardDifference"
self.subconfig4.forwardRegularizationOrders = [1, 2]
self.kList1 = ipDiffim.makeKernelBasisList(self.subconfig1)
self.bskv1 = ipDiffim.BuildSingleKernelVisitorF(self.kList1, pexConfig.makePolicy(self.subconfig1))
self.kList2 = ipDiffim.makeKernelBasisList(self.subconfig2)
self.hMat2 = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfig2))
self.bskv2 = ipDiffim.BuildSingleKernelVisitorF(self.kList2,
pexConfig.makePolicy(self.subconfig2), self.hMat2)
self.kList3 = ipDiffim.makeKernelBasisList(self.subconfig3)
self.hMat3 = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfig3))
self.bskv3 = ipDiffim.BuildSingleKernelVisitorF(self.kList3,
pexConfig.makePolicy(self.subconfig3), self.hMat3)
self.kList4 = ipDiffim.makeKernelBasisList(self.subconfig4)
self.hMat4 = ipDiffim.makeRegularizationMatrix(pexConfig.makePolicy(self.subconfig4))
self.bskv4 = ipDiffim.BuildSingleKernelVisitorF(self.kList4,
pexConfig.makePolicy(self.subconfig4), self.hMat4)
# known input images
defDataDir = lsst.utils.getPackageDir('afwdata')
if CFHT:
defSciencePath = os.path.join(defDataDir, 'CFHT', 'D4', CFHTTORUN+'.fits')
defTemplatePath = os.path.join(defDataDir, 'CFHT', 'D4', CFHTTORUN+'_tmpl.fits')
# no need to remap
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
else:
defSciencePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v26-e0",
"v26-e0-c011-a00.sci")
defTemplatePath = os.path.join(defDataDir, "DC3a-Sim", "sci", "v5-e0",
"v5-e0-c011-a00.sci")
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
warper = afwMath.Warper.fromConfig(self.subconfig1.warpingConfig)
self.templateExposure = warper.warpExposure(self.scienceExposure.getWcs(), self.templateExposure,
destBBox=self.scienceExposure.getBBox())
diffimTools.backgroundSubtract(self.subconfig1.afwBackgroundConfig,
[self.scienceExposure.getMaskedImage(),
self.templateExposure.getMaskedImage()])
#
tmi = self.templateExposure.getMaskedImage()
smi = self.scienceExposure.getMaskedImage()
detConfig = self.subconfig1.detectionConfig
detPolicy = pexConfig.makePolicy(detConfig)
detPolicy.set("detThreshold", 50.)
detPolicy.set("detOnTemplate", False)
kcDetect = ipDiffim.KernelCandidateDetectionF(detPolicy)
kcDetect.apply(tmi, smi)
self.footprints = kcDetect.getFootprints()