def testTablePersistence(self):
filename = "SingleGaussianPsf.fits"
psf1 = algorithms.SingleGaussianPsf(5, 7, 4.2)
psf1.writeFits(filename)
psf2 = algorithms.SingleGaussianPsf.readFits(filename)
self.assertEqual(psf1.getSigma(), psf2.getSigma())
os.remove(filename)
def setUp(self):
FWHM = 5
self.ksize = 25 # size of desired kernel
sigma = FWHM / (2 * math.sqrt(2 * math.log(2)))
self.psfDg = measAlg.DoubleGaussianPsf(self.ksize, self.ksize, sigma,
1, 0.1)
self.psfSg = measAlg.SingleGaussianPsf(self.ksize, self.ksize, sigma)
def loadData(imFile=None):
"""Prepare the data we need to run the example"""
if imFile is None:
# Load sample input from disk
afwdataDir = lsst.utils.getPackageDir('afwdata')
imFile = os.path.join(afwdataDir, "CFHT", "D4", "cal-53535-i-797722_small_1.fits")
else:
if not os.path.isfile(imFile):
print("Input file %s does not exist" % (imFile), file=sys.stderr)
sys.exit(1)
exposure = afwImage.ExposureF(imFile)
psf = measAlg.SingleGaussianPsf(21, 21, 2)
exposure.setPsf(psf)
im = exposure.getMaskedImage().getImage()
im -= np.nanmedian(im.getArray())
# Create the dipole
offset = 3
tmpim = im.getArray()[:-offset, :-offset]
im.getArray()[offset:, offset:] -= tmpim
return exposure
def test(self):
schema = afwTable.ExposureTable.makeMinimalSchema()
schema.addField("ccd", np.int32, doc="CCD number")
schema.addField("visit", np.int32, doc="Visit number")
schema.addField("goodpix", np.int32, doc="Number of good pixels")
schema.addField("weight", float, doc="Weighting for this CCD")
ccds = afwTable.ExposureCatalog(schema)
wcs = afwImage.makeWcs(afwCoord.Coord(0.0*afwGeom.degrees, 0.0*afwGeom.degrees),
afwGeom.Point2D(0.0, 0.0), 1.0e-4, 0.0, 0.0, 1.0e-4)
new = ccds.addNew()
new.set("id", 0)
new.set("bbox_min_x", 0)
new.set("bbox_min_y", 0)
new.set("bbox_max_x", 1024)
new.set("bbox_max_y", 1024)
# The following lines are critical for reproducing the bug, because
# the code is reading a double starting at the 'ccd' (offset 24), and
# it sees a zero (from the zero in 'ccd' and the leading zeros in 'visit').
new.set("ccd", 0)
new.set("visit", 6789)
new.set("goodpix", 987654321)
new.set("weight", 1.0)
new.setPsf(measAlg.SingleGaussianPsf(23, 23, 2.345))
new.setWcs(wcs)
# In the presence of the bug, the following fails with
# lsst::pex::exceptions::RuntimeError thrown in src/CoaddPsf.cc
# with message: "Could not find a valid average position for CoaddPsf"
measAlg.CoaddPsf(ccds, wcs)
def test(self):
schema = afwTable.ExposureTable.makeMinimalSchema()
schema.addField("ccd", int, doc="CCD number")
schema.addField("visit", long, doc="Visit number")
schema.addField("goodpix", int, doc="Number of good pixels")
schema.addField("weight", float, doc="Weighting for this CCD")
ccds = afwTable.ExposureCatalog(schema)
wcs = afwImage.makeWcs(afwCoord.Coord(0.0*afwGeom.degrees, 0.0*afwGeom.degrees),
afwGeom.Point2D(0.0, 0.0), 1.0e-4, 0.0, 0.0, 1.0e-4)
new = ccds.addNew()
new.set("id", 0)
new.set("bbox.min", afwGeom.Point2I(0,0))
new.set("bbox.max", afwGeom.Point2I(1024,1024))
# The following lines are critical for reproducing the bug, because
# the code is reading a double starting at the 'ccd' (offset 24), and
# it sees a zero (from the zero in 'ccd' and the leading zeros in 'visit').
new.set("ccd", 0)
new.set("visit", 6789)
new.set("goodpix", 987654321)
new.set("weight", 1.0)
new.setPsf(measAlg.SingleGaussianPsf(23, 23, 2.345))
new.setWcs(wcs)
# In the presence of the bug, the following fails with:
# LsstCppException: 0: lsst::pex::exceptions::RuntimeErrorException thrown at src/CoaddPsf.cc:134 in lsst::afw::geom::Point2D lsst::meas::algorithms::{anonymous}::computeAveragePosition(const ExposureCatalog&, const lsst::afw::image::Wcs&, lsst::afw::table::Key<double>)
# 0: Message: Could not find a valid average position for CoaddPsf
measAlg.CoaddPsf(ccds, wcs)
def testInvalidSgPsf(self):
"""Test parameters of sgPsfs, both valid and not."""
sigma = 1.
measAlg.SingleGaussianPsf(self.ksize, self.ksize, sigma)
def badSigma1():
sigma = 0
measAlg.SingleGaussianPsf(self.ksize, self.ksize, sigma)
with self.assertRaises(pexExceptions.DomainError):
badSigma1()
def loadData():
"""Prepare the data we need to run the example"""
# Load sample input from disk
afwdataDir = lsst.utils.getPackageDir('afwdata')
imFile = os.path.join(afwdataDir, "CFHT", "D4", "cal-53535-i-797722_small_1.fits")
exposure = afwImage.ExposureF(imFile)
psf = measAlg.SingleGaussianPsf(21, 21, 2)
exposure.setPsf(psf)
im = exposure.getMaskedImage().getImage()
im -= float(np.median(im.getArray()))
return exposure
def crImageStats(self, exposure, statControl):
"""Measure amplifier level statistics on the exposure,
after running cosmic ray rejection.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
The exposure to measure.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object with parameters defined by
the config.
Returns
-------
outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
A dictionary indexed by the amplifier name, containing
dictionaries of the statistics measured and their values.
"""
crRejectedExp = exposure.clone()
psf = measAlg.SingleGaussianPsf(
self.config.psfSize, self.config.psfSize,
self.config.psfFwhm / (2 * math.sqrt(2 * math.log(2))))
crRejectedExp.setPsf(psf)
try:
self.repair.run(crRejectedExp, keepCRs=False)
failAll = False
except pexException.LengthError:
self.log.warning(
"Failure masking cosmic rays (too many found). Continuing.")
failAll = True
if self.config.crGrow > 0:
crMask = crRejectedExp.getMaskedImage().getMask().getPlaneBitMask(
"CR")
spans = afwGeom.SpanSet.fromMask(crRejectedExp.mask, crMask)
spans = spans.dilated(self.config.crGrow)
spans = spans.clippedTo(crRejectedExp.getBBox())
spans.setMask(crRejectedExp.mask, crMask)
return self.amplifierStats(crRejectedExp,
self.config.crImageStatKeywords,
statControl,
failAll=failAll)
def loadData():
"""Prepare the data we need to run the example"""
# Load sample input from disk
mypath = eups.productDir("afwdata")
if not mypath:
print >> sys.stderr, "Please setup afwdata and try again"
sys.exit(1)
imFile = os.path.join(mypath, "CFHT", "D4",
"cal-53535-i-797722_small_1.fits")
exposure = afwImage.ExposureF(imFile)
psf = measAlg.SingleGaussianPsf(21, 21, 2)
exposure.setPsf(psf)
im = exposure.getMaskedImage().getImage()
im -= float(numpy.median(im.getArray()))
return exposure
def test_amplifierSigma(self):
"""Clipped sigma against CR-rejected sigma
Notes
-----
DMTN-101 4.4
Run a CR rejection on the result and confirm that the
unclipped standard deviation is consistent with the 5-sigma
clipped value.
"""
crTask = RepairTask()
crRejected = self.exposure.clone()
psf = measAlg.SingleGaussianPsf(21, 21, 3.0)
crRejected.setPsf(psf)
crTask.run(crRejected, keepCRs=False)
ccd = self.exposure.getDetector()
for amp in ccd:
ampExposure = self.exposure.Factory(self.exposure, amp.getBBox())
clipControl = afwMath.StatisticsControl(5.0, 5)
clipControl.setAndMask(
self.exposure.mask.getPlaneBitMask(["SAT", "BAD", "NO_DATA"]))
sigmaClip = afwMath.makeStatistics(ampExposure.getImage(),
afwMath.STDEVCLIP,
clipControl).getValue()
crAmp = crRejected.Factory(crRejected, amp.getBBox())
statControl = afwMath.StatisticsControl()
statControl.setAndMask(
self.exposure.mask.getPlaneBitMask(
["SAT", "BAD", "NO_DATA", "CR"]))
sigma = afwMath.makeStatistics(crAmp.getImage(), afwMath.STDEV,
statControl).getValue()
# needs to be < 0.05
fractionalError = np.abs(sigma - sigmaClip) / sigmaClip
self.assertLess(fractionalError,
3.0,
msg="Test 4.4: {amp.getName()} {fractionalError}")
def loadData():
"""Prepare the data we need to run the example"""
# Load sample input from disk
try:
mypath = getPackageDir("afwdata")
except pexExcept.NotFoundError:
print("Please setup afwdata and try again", file=sys.stderr)
sys.exit(1)
imFile = os.path.join(mypath, "CFHT", "D4",
"cal-53535-i-797722_small_1.fits")
exposure = afwImage.ExposureF(imFile)
psf = measAlg.SingleGaussianPsf(21, 21, 2)
exposure.setPsf(psf)
im = exposure.getMaskedImage().getImage()
im -= float(np.median(im.getArray()))
return exposure
def run(self, inputExp):
"""Preprocess input exposures prior to DARK combination.
This task detects and repairs cosmic rays strikes.
Parameters
----------
inputExp : `lsst.afw.image.Exposure`
Pre-processed dark frame data to combine.
Returns
-------
outputExp : `lsst.afw.image.Exposure`
CR rejected, ISR processed Dark Frame."
"""
psf = measAlg.SingleGaussianPsf(
self.config.psfSize, self.config.psfSize,
self.config.psfFwhm / (2 * math.sqrt(2 * math.log(2))))
inputExp.setPsf(psf)
scaleExp = inputExp.clone()
mi = scaleExp.getMaskedImage()
# DM-23680:
# Darktime scaling necessary for repair.run() to ID CRs correctly.
scale = inputExp.getInfo().getVisitInfo().getDarkTime()
if np.isfinite(scale) and scale != 0.0:
mi /= scale
self.repair.run(scaleExp, keepCRs=False)
if self.config.crGrow > 0:
crMask = inputExp.getMaskedImage().getMask().getPlaneBitMask("CR")
spans = afwGeom.SpanSet.fromMask(inputExp.mask, crMask)
spans = spans.dilated(self.config.crGrow)
spans.setMask(inputExp.mask, crMask)
# Undo scaling; as above, DM-23680.
if np.isfinite(scale) and scale != 0.0:
mi *= scale
return pipeBase.Struct(outputExp=inputExp, )
def testGaussian(self):
"""Check that we can measure a single Gaussian's attributes"""
sigma0 = 5.0
aEff0 = 4.0 * math.pi * sigma0**2
xwid = int(12 * sigma0)
ywid = xwid
# set the peak of the outer guassian to 0 so this is really a single gaussian.
psf = measAlg.SingleGaussianPsf(xwid, ywid, sigma0)
if False and display:
im = psf.computeImage(afwGeom.PointD(xwid // 2, ywid // 2))
ds9.mtv(im, title="N(%g) psf" % sigma0, frame=0)
psfAttrib = measAlg.PsfAttributes(psf, xwid // 2, ywid // 2)
sigma = psfAttrib.computeGaussianWidth(psfAttrib.ADAPTIVE_MOMENT)
m1 = psfAttrib.computeGaussianWidth(psfAttrib.FIRST_MOMENT)
m2 = psfAttrib.computeGaussianWidth(psfAttrib.SECOND_MOMENT)
noise = psfAttrib.computeGaussianWidth(psfAttrib.NOISE_EQUIVALENT)
bick = psfAttrib.computeGaussianWidth(psfAttrib.BICKERTON)
aEff = psfAttrib.computeEffectiveArea()
if verbose:
print "Adaptive %g v %g" % (sigma0, sigma)
print "First moment %g v %g" % (sigma0, m1)
print "Second moment %g v %g" % (sigma0, m2)
print "Noise Equivalent %g v %g" % (sigma0, sigma)
print "Bickerton %g v %g" % (sigma0, bick)
print "Effective area %g v %f" % (aEff0, aEff)
self.assertTrue(abs(sigma0 - sigma) <= 1.0e-2)
self.assertTrue(abs(sigma0 - m1) <= 3.0e-2)
self.assertTrue(abs(sigma0 - m2) <= 1.0e-2)
self.assertTrue(abs(sigma0 - noise) <= 1.0e-2)
self.assertTrue(abs(sigma0 - bick) <= 1.0e-2)
self.assertTrue(abs(aEff0 - aEff) <= 1.0e-2)
def testDistortedImage(self):
detector = self.detector
psfSigma = 1.5
stars = plantSources(self.x0, self.y0, self.nx, self.ny, self.sky, self.nObj, psfSigma, detector)
expos, starXy = stars[0], stars[1]
# add some faint round galaxies ... only slightly bigger than the psf
gxy = plantSources(self.x0, self.y0, self.nx, self.ny, self.sky, 10, 1.07*psfSigma, detector)
mi = expos.getMaskedImage()
mi += gxy[0].getMaskedImage()
gxyXy = gxy[1]
kwid = 15 #int(10*psfSigma) + 1
psf = measAlg.SingleGaussianPsf(kwid, kwid, psfSigma)
expos.setPsf(psf)
expos.setDetector(detector)
########################
# try without distorter
expos.setDetector(self.flatDetector)
print "Testing PSF selection *without* distortion"
sourceList = self.detectAndMeasure(expos)
psfCandidateList = self.starSelector.run(expos, sourceList).psfCandidates
########################
# try with distorter
expos.setDetector(self.detector)
print "Testing PSF selection *with* distortion"
sourceList = self.detectAndMeasure(expos)
psfCandidateListCorrected = self.starSelector.run(expos, sourceList).psfCandidates
def countObjects(candList):
nStar, nGxy = 0, 0
for c in candList:
s = c.getSource()
x, y = s.getX(), s.getY()
for xs,ys in starXy:
if abs(x-xs) < 2.0 and abs(y-ys) < 2.0:
nStar += 1
for xg,yg in gxyXy:
if abs(x-xg) < 2.0 and abs(y-yg) < 2.0:
nGxy += 1
return nStar, nGxy
nstar, ngxy = countObjects(psfCandidateList)
nstarC, ngxyC = countObjects(psfCandidateListCorrected)
print "uncorrected nStar, nGxy: ", nstar, "/", len(starXy)," ", ngxy, '/', len(gxyXy)
print "dist-corrected nStar, nGxy: ", nstarC, '/', len(starXy)," ", ngxyC, '/', len(gxyXy)
########################
# display
if display:
iDisp = 1
ds9.mtv(expos, frame=iDisp)
size = 40
for c in psfCandidateList:
s = c.getSource()
ixx, iyy, ixy = size*s.getIxx(), size*s.getIyy(), size*s.getIxy()
ds9.dot("@:%g,%g,%g" % (ixx, ixy, iyy), s.getX(), s.getY(),
frame=iDisp, ctype=ds9.RED)
size *= 2.0
for c in psfCandidateListCorrected:
s = c.getSource()
ixx, iyy, ixy = size*s.getIxx(), size*s.getIyy(), size*s.getIxy()
ds9.dot("@:%g,%g,%g" % (ixx, ixy, iyy), s.getX(), s.getY(),
frame=iDisp, ctype=ds9.GREEN)
# we shouldn't expect to get all available stars without distortion correcting
self.assertLess(nstar, len(starXy))
# here we should get all of them, occassionally 1 or 2 might get missed
self.assertGreaterEqual(nstarC, 0.95*len(starXy))
# no contamination by small gxys
self.assertEqual(ngxyC, 0)
def testPsfCandidate(self):
detector = self.detector
# make an exposure
print "Planting"
psfSigma = 1.5
exposDist, nGoodDist, expos0, nGood0 = plantSources(self.x0, self.y0,
self.nx, self.ny,
self.sky, self.nObj, psfSigma, detector)
# set the psf
kwid = 21
psf = measAlg.SingleGaussianPsf(kwid, kwid, psfSigma)
exposDist.setPsf(psf)
exposDist.setDetector(detector)
# detect
print "detection"
sourceList = self.detectAndMeasure(exposDist)
# select psf stars
print "PSF selection"
psfCandidateList = self.starSelector.run(exposDist, sourceList).psfCandidates
# determine the PSF
print "PSF determination"
metadata = dafBase.PropertyList()
t0 = time.time()
psf, cellSet = self.psfDeterminer.determinePsf(exposDist, psfCandidateList, metadata)
print "... determination time: ", time.time() - t0
print "PSF kernel width: ", psf.getKernel().getWidth()
#######################################################################
# try to subtract off the stars and check the residuals
imgOrig = exposDist.getMaskedImage().getImage().getArray()
maxFlux = imgOrig.max()
############
# first try it with no distortion in the psf
exposDist.setDetector(self.flatDetector)
print "uncorrected subtraction"
subImg = afwImage.MaskedImageF(exposDist.getMaskedImage(), True)
for s in sourceList:
x, y = s.getX(), s.getY()
measAlg.subtractPsf(psf, subImg, x, y)
if display:
settings = {'scale': 'minmax', 'zoom':"to fit", 'mask':'transparency 80'}
ds9.mtv(exposDist, frame=1, title="full", settings=settings)
ds9.mtv(subImg, frame=2, title="subtracted", settings=settings)
img = subImg.getImage().getArray()
norm = img/math.sqrt(maxFlux)
smin0, smax0, srms0 = norm.min(), norm.max(), norm.std()
print "min:", smin0, "max: ", smax0, "rms: ", srms0
if False:
# This section has been disabled as distortion was removed from PsfCandidate and Psf;
# it will be reintroduced in the future with a different API, at which point this
# test code should be re-enabled.
##############
# try it with the correct distortion in the psf
exposDist.setDetector(self.detector)
print "corrected subtraction"
subImg = afwImage.MaskedImageF(exposDist.getMaskedImage(), True)
for s in sourceList:
x, y = s.getX(), s.getY()
measAlg.subtractPsf(psf, subImg, x, y)
if display:
settings = {'scale': 'minmax', 'zoom':"to fit", 'mask':'transparency 80'}
ds9.mtv(exposDist, frame=1, title="full", settings=settings)
ds9.mtv(subImg, frame=2, title="subtracted", settings=settings)
img = subImg.getImage().getArray()
norm = img/math.sqrt(maxFlux)
smin, smax, srms = norm.min(), norm.max(), norm.std()
# with proper distortion, residuals should be < 4sigma (even for 512x512 pixels)
print "min:", smin, "max: ", smax, "rms: ", srms
# the distrib of residuals should be tighter
self.assertLess(smin0, smin)
self.assertGreater(smax0, smax)
self.assertGreater(srms0, srms)
def testPeakLikelihoodFlux(self):
"""Test measurement with PeakLikelihoodFlux
"""
# make mp: a flux measurer
measControl = measAlg.PeakLikelihoodFluxControl()
schema = afwTable.SourceTable.makeMinimalSchema()
mp = measAlg.MeasureSourcesBuilder().addAlgorithm(measControl).build(
schema)
# make and measure a series of exposures containing just one star, approximately centered
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(100, 101))
kernelWidth = 35
var = 100
fwhm = 3.0
sigma = fwhm / FwhmPerSigma
convolutionControl = afwMath.ConvolutionControl()
psf = measAlg.SingleGaussianPsf(kernelWidth, kernelWidth, sigma)
psfKernel = psf.getLocalKernel()
psfImage = psf.computeKernelImage()
sumPsfSq = numpy.sum(psfImage.getArray()**2)
psfSqArr = psfImage.getArray()**2
for flux in (1000, 10000):
ctrInd = afwGeom.Point2I(50, 51)
ctrPos = afwGeom.Point2D(ctrInd)
kernelBBox = psfImage.getBBox(afwImage.PARENT)
kernelBBox.shift(afwGeom.Extent2I(ctrInd))
# compute predicted flux error
unshMImage = makeFakeImage(bbox, [ctrPos], [flux], fwhm, var)
# filter image by PSF
unshFiltMImage = afwImage.MaskedImageF(
unshMImage.getBBox(afwImage.PARENT))
afwMath.convolve(unshFiltMImage, unshMImage, psfKernel,
convolutionControl)
# compute predicted flux = value of image at peak / sum(PSF^2)
# this is a sanity check of the algorithm, as much as anything
predFlux = unshFiltMImage.getImage().get(ctrInd[0],
ctrInd[1]) / sumPsfSq
self.assertLess(abs(flux - predFlux), flux * 0.01)
# compute predicted flux error based on filtered pixels
# = sqrt(value of filtered variance at peak / sum(PSF^2)^2)
predFluxErr = math.sqrt(unshFiltMImage.getVariance().get(
ctrInd[0], ctrInd[1])) / sumPsfSq
# compute predicted flux error based on unfiltered pixels
# = sqrt(sum(unfiltered variance * PSF^2)) / sum(PSF^2)
# and compare to that derived from filtered pixels;
# again, this is a test of the algorithm
varView = afwImage.ImageF(unshMImage.getVariance(), kernelBBox)
varArr = varView.getArray()
unfiltPredFluxErr = math.sqrt(numpy.sum(
varArr * psfSqArr)) / sumPsfSq
self.assertLess(abs(unfiltPredFluxErr - predFluxErr),
predFluxErr * 0.01)
for fracOffset in (afwGeom.Extent2D(0, 0),
afwGeom.Extent2D(0.2, -0.3)):
adjCenter = ctrPos + fracOffset
if fracOffset == (0, 0):
maskedImage = unshMImage
filteredImage = unshFiltMImage
else:
maskedImage = makeFakeImage(bbox, [adjCenter], [flux],
fwhm, var)
# filter image by PSF
filteredImage = afwImage.MaskedImageF(
maskedImage.getBBox(afwImage.PARENT))
afwMath.convolve(filteredImage, maskedImage, psfKernel,
convolutionControl)
exposure = afwImage.makeExposure(filteredImage)
exposure.setPsf(psf)
table = afwTable.SourceTable.make(schema)
source = table.makeRecord()
mp.apply(source, exposure, afwGeom.Point2D(*adjCenter))
measFlux = source.get(measControl.name)
measFluxErr = source.get(measControl.name + ".err")
self.assertFalse(source.get(measControl.name + ".flags"))
self.assertLess(abs(measFlux - flux), flux * 0.003)
self.assertLess(abs(measFluxErr - predFluxErr),
predFluxErr * 0.2)
# try nearby points and verify that the flux is smaller;
# this checks that the sub-pixel shift is performed in the correct direction
for dx in (-0.2, 0, 0.2):
for dy in (-0.2, 0, 0.2):
if dx == dy == 0:
continue
offsetCtr = afwGeom.Point2D(adjCenter[0] + dx,
adjCenter[1] + dy)
table = afwTable.SourceTable.make(schema)
source = table.makeRecord()
mp.apply(source, exposure, offsetCtr)
offsetFlux = source.get(measControl.name)
self.assertLess(offsetFlux, measFlux)
# source so near edge of image that PSF does not overlap exposure should result in failure
for edgePos in (
(1, 50),
(50, 1),
(50, bbox.getHeight() - 1),
(bbox.getWidth() - 1, 50),
):
table = afwTable.SourceTable.make(schema)
source = table.makeRecord()
mp.apply(source, exposure, afwGeom.Point2D(*edgePos))
self.assertTrue(source.get(measControl.name + ".flags"))
# no PSF should result in failure: flags set
noPsfExposure = afwImage.ExposureF(filteredImage)
table = afwTable.SourceTable.make(schema)
source = table.makeRecord()
mp.apply(source, noPsfExposure, afwGeom.Point2D(*adjCenter))
self.assertTrue(source.get(measControl.name + ".flags"))
def badSigma1():
sigma = 0
measAlg.SingleGaussianPsf(self.ksize, self.ksize, sigma)