def testFilters(self):
"""Test that the coadd filter is set correctly
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw", "SdssFilters.paf", "tests")
filterPolicy = pexPolicy.Policy.createPolicy(filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
unkFilter = afwImage.Filter()
gFilter = afwImage.Filter("g")
rFilter = afwImage.Filter("r")
inExp = afwImage.ExposureF(ImSimFile, afwGeom.Box2I(afwGeom.Point2I(0,0), afwGeom.Extent2I(10, 10)),
afwImage.PARENT)
coadd = coaddUtils.Coadd(
bbox = inExp.getBBox(),
wcs = inExp.getWcs(),
badMaskPlanes = ("NO_DATA", "BAD"),
)
inExp.setFilter(gFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter,))
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter,))
inExp.setFilter(rFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), unkFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, rFilter))
def testNullWarpExposure(self, interpLength=10):
"""Test that warpExposure maps an image onto itself.
Note:
- NO_DATA and off-CCD pixels must be ignored
- bad mask pixels get smeared out so we have to excluded all bad mask pixels
from the output image when comparing masks.
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalExposure = afwImage.ExposureF(originalExposurePath)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
afwWarpedExposure = afwImage.ExposureF(originalExposure.getBBox(),
originalExposure.getWcs())
warpingControl = afwMath.WarpingControl("lanczos4", "", 0,
interpLength)
afwMath.warpExposure(afwWarpedExposure, originalExposure,
warpingControl)
if SAVE_FITS_FILES:
afwWarpedExposure.writeFits("afwWarpedExposureNull.fits")
self.assertEquals(afwWarpedExposure.getFilter().getName(),
originalFilter.getName())
self.assertEquals(afwWarpedExposure.getCalib().getFluxMag0(),
originalCalib.getFluxMag0())
afwWarpedMaskedImage = afwWarpedExposure.getMaskedImage()
afwWarpedMask = afwWarpedMaskedImage.getMask()
noDataBitMask = afwWarpedMask.getPlaneBitMask("NO_DATA")
afwWarpedMaskedImageArrSet = afwWarpedMaskedImage.getArrays()
afwWarpedMaskArr = afwWarpedMaskedImageArrSet[1]
# compare all non-DATA pixels of image and variance, but relax specs a bit
# because of minor noise introduced by bad pixels
noDataMaskArr = afwWarpedMaskArr & noDataBitMask
msg = "afw null-warped MaskedImage (all pixels, relaxed tolerance)"
self.assertMaskedImagesNearlyEqual(afwWarpedMaskedImage,
originalExposure.getMaskedImage(),
doMask=False,
skipMask=noDataMaskArr,
atol=1e-5,
msg=msg)
# compare good pixels (mask=0) of image, mask and variance using full tolerance
msg = "afw null-warped MaskedImage (good pixels, max tolerance)"
self.assertMaskedImagesNearlyEqual(afwWarpedMaskedImage,
originalExposure.getMaskedImage(),
skipMask=afwWarpedMask,
msg=msg)
def testNullWarpExposure(self, interpLength=10):
"""Test that warpExposure maps an image onto itself.
Note:
- edge and off-CCD pixels must be ignored
- bad mask pixels get smeared out so we have to excluded all bad mask pixels
from the output image when comparing masks.
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw", "SdssFilters.paf", "tests")
filterPolicy = pexPolicy.Policy.createPolicy(filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalExposure = afwImage.ExposureF(originalExposurePath)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
afwWarpedExposure = afwImage.ExposureF(
originalExposure.getBBox(),
originalExposure.getWcs())
warpingControl = afwMath.WarpingControl("lanczos4", "", 0, interpLength)
afwMath.warpExposure(afwWarpedExposure, originalExposure, warpingControl)
if SAVE_FITS_FILES:
afwWarpedExposure.writeFits("afwWarpedExposureNull.fits")
self.assertEquals(afwWarpedExposure.getFilter().getName(), originalFilter.getName())
self.assertEquals(afwWarpedExposure.getCalib().getFluxMag0(), originalCalib.getFluxMag0())
afwWarpedMaskedImage = afwWarpedExposure.getMaskedImage()
afwWarpedMask = afwWarpedMaskedImage.getMask()
edgeBitMask = afwWarpedMask.getPlaneBitMask("EDGE")
noDataBitMask = afwWarpedMask.getPlaneBitMask("NO_DATA")
if edgeBitMask == 0:
self.fail("warped mask has no EDGE bit")
if noDataBitMask == 0:
self.fail("warped mask has no NO_DATA bit")
afwWarpedMaskedImageArrSet = afwWarpedMaskedImage.getArrays()
afwWarpedMaskArr = afwWarpedMaskedImageArrSet[1]
# compare all non-edge pixels of image and variance, but relax specs a bit
# because of minor noise introduced by bad pixels
maskArr = afwWarpedMaskArr & (edgeBitMask | noDataBitMask)
originalMaskedImageArrSet = originalExposure.getMaskedImage().getArrays()
errStr = imageTestUtils.maskedImagesDiffer(afwWarpedMaskedImageArrSet, originalMaskedImageArrSet,
doMask=False, skipMaskArr=maskArr, atol=1e-5)
if errStr:
self.fail("afw null-warped MaskedImage (all pixels, relaxed tolerance): %s" % (errStr,))
# compare good pixels of image, mask and variance using full tolerance
errStr = imageTestUtils.maskedImagesDiffer(afwWarpedMaskedImageArrSet, originalMaskedImageArrSet,
doImage=False, doVariance=False, skipMaskArr=afwWarpedMaskArr)
if errStr:
self.fail("afw null-warped MaskedImage (good pixels, max tolerance): %s" % (errStr,))
def setUp(self):
# Initialise our filters
#
# Start by forgetting that we may already have defined filters
#
filterPolicy = pexPolicy.Policy.createPolicy(
os.path.join(eups.productDir("afw"), "tests", "SdssFilters.paf"), True)
self.filters = tuple(sorted([f.get("name") for f in filterPolicy.getArray("Filter")]))
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
self.g_lambdaEff = [p.get("lambdaEff") for p in filterPolicy.getArray("Filter")
if p.get("name") == "g"][0] # used for tests
def testBBox(self):
"""Test that the default bounding box includes all warped pixels
"""
kernelName = "lanczos2"
warper = afwMath.Warper(kernelName)
originalExposure, swarpedImage, swarpedWcs = self.getSwarpedImage(
kernelName=kernelName, useSubregion=True, useDeepCopy=False)
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
warpedExposure1 = warper.warpExposure(destWcs=swarpedWcs,
srcExposure=originalExposure)
# the default size must include all good pixels, so growing the bbox
# should not add any
warpedExposure2 = warper.warpExposure(destWcs=swarpedWcs,
srcExposure=originalExposure,
border=1)
# a bit of excess border is allowed, but surely not as much as 10 (in
# fact it is approx. 5)
warpedExposure3 = warper.warpExposure(destWcs=swarpedWcs,
srcExposure=originalExposure,
border=-10)
# assert that warpedExposure and warpedExposure2 have the same number of non-no_data pixels
# and that warpedExposure3 has fewer
noDataBitMask = afwImage.Mask.getPlaneBitMask("NO_DATA")
mask1Arr = warpedExposure1.getMaskedImage().getMask().getArray()
mask2Arr = warpedExposure2.getMaskedImage().getMask().getArray()
mask3Arr = warpedExposure3.getMaskedImage().getMask().getArray()
nGood1 = (mask1Arr & noDataBitMask == 0).sum()
nGood2 = (mask2Arr & noDataBitMask == 0).sum()
nGood3 = (mask3Arr & noDataBitMask == 0).sum()
self.assertEqual(nGood1, nGood2)
self.assertLess(nGood3, nGood1)
self.assertEqual(warpedExposure1.getFilter().getName(),
originalFilter.getName())
self.assertEqual(warpedExposure1.getCalib().getFluxMag0(),
originalCalib.getFluxMag0())
def testFilters(self):
"""Test that the coadd filter is set correctly
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
unkFilter = afwImage.Filter()
gFilter = afwImage.Filter("g")
rFilter = afwImage.Filter("r")
imShape = (150, 150)
imSigma = 1.0
imVariance = 1.0
badMaskPlanes = ["EDGE"]
np.random.seed(0)
coadd = None
maskedImage = afwTestUtils.makeGaussianNoiseMaskedImage(
dimensions=imShape, sigma=imSigma, variance=imVariance)
inExp = afwImage.ExposureF(maskedImage)
coadd = coaddChiSq.Coadd(
bbox=inExp.getBBox(),
wcs=inExp.getWcs(),
badMaskPlanes=badMaskPlanes,
)
inExp.setFilter(gFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, ))
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, ))
inExp.setFilter(rFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), unkFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, rFilter))
def testFilters(self):
"""Test that the coadd filter is set correctly
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw", "SdssFilters.paf", "tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
unkFilter = afwImage.Filter()
gFilter = afwImage.Filter("g")
rFilter = afwImage.Filter("r")
imShape = (150, 150)
imSigma = 1.0
imVariance = 1.0
badMaskPlanes = ["EDGE"]
np.random.seed(0)
coadd = None
maskedImage = afwTestUtils.makeGaussianNoiseMaskedImage(
dimensions=imShape, sigma=imSigma, variance=imVariance)
inExp = afwImage.ExposureF(maskedImage)
coadd = coaddChiSq.Coadd(
bbox=inExp.getBBox(),
wcs=inExp.getWcs(),
badMaskPlanes=badMaskPlanes,
)
inExp.setFilter(gFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter,))
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter,))
inExp.setFilter(rFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), unkFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, rFilter))
def testBBox(self):
"""Test that the default bounding box includes all warped pixels
"""
kernelName = "lanczos2"
warper = afwMath.Warper(kernelName)
originalExposure, swarpedImage, swarpedWcs = self.getSwarpedImage(
kernelName=kernelName, useSubregion=True, useDeepCopy=False)
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw", "SdssFilters.paf", "tests")
filterPolicy = pexPolicy.Policy.createPolicy(filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
warpedExposure1 = warper.warpExposure(destWcs=swarpedWcs, srcExposure=originalExposure)
# the default size must include all good pixels, so growing the bbox should not add any
warpedExposure2 = warper.warpExposure(destWcs=swarpedWcs, srcExposure=originalExposure, border=1)
# a bit of excess border is allowed, but surely not as much as 10 (in fact it is approx. 5)
warpedExposure3 = warper.warpExposure(destWcs=swarpedWcs, srcExposure=originalExposure, border=-10)
# assert that warpedExposure and warpedExposure2 have the same number of non-edge pixels
# and that warpedExposure3 has fewer
edgeMask = 1 << afwImage.MaskU.getMaskPlane("EDGE")
mask1Arr = warpedExposure1.getMaskedImage().getMask().getArray()
mask2Arr = warpedExposure2.getMaskedImage().getMask().getArray()
mask3Arr = warpedExposure3.getMaskedImage().getMask().getArray()
nGood1 = (mask1Arr & edgeMask == 0).sum()
nGood2 = (mask2Arr & edgeMask == 0).sum()
nGood3 = (mask3Arr & edgeMask == 0).sum()
self.assertTrue(nGood1 == nGood2)
self.assertTrue(nGood3 < nGood1)
self.assertEquals(warpedExposure1.getFilter().getName(), originalFilter.getName())
self.assertEquals(warpedExposure1.getCalib().getFluxMag0(), originalCalib.getFluxMag0())
def testFilters(self):
"""Test that the coadd filter is set correctly
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
unkFilter = afwImage.Filter()
gFilter = afwImage.Filter("g")
rFilter = afwImage.Filter("r")
calexpPath = os.path.join(AfwdataDir, SimCalexpSubpath)
inExp = afwImage.ExposureF(
calexpPath,
afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(10, 10)),
afwImage.PARENT)
coadd = coaddUtils.Coadd(
bbox=inExp.getBBox(),
wcs=inExp.getWcs(),
badMaskPlanes=("NO_DATA", "BAD"),
)
inExp.setFilter(gFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, ))
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), gFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, ))
inExp.setFilter(rFilter)
coadd.addExposure(inExp)
self.assertEqualFilters(coadd.getCoadd().getFilter(), unkFilter)
self.assertEqualFilterSets(coadd.getFilters(), (gFilter, rFilter))
def setUp(self):
# Initialise our filters
filterPolicy = pexPolicy.Policy.createPolicy(
os.path.join(eups.productDir("afw"), "tests", "SdssFilters.paf"), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
def testNullWarpExposure(self, interpLength=10):
"""Test that warpExposure maps an image onto itself.
Note:
- edge pixels must be ignored
- bad mask pixels get smeared out so we have to excluded all bad mask pixels
from the output image when comparing masks.
"""
filterPolicyFile = pexPolicy.DefaultPolicyFile("afw",
"SdssFilters.paf",
"tests")
filterPolicy = pexPolicy.Policy.createPolicy(
filterPolicyFile, filterPolicyFile.getRepositoryPath(), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
originalExposure = afwImage.ExposureF(originalExposurePath)
originalFilter = afwImage.Filter("i")
originalCalib = afwImage.Calib()
originalCalib.setFluxMag0(1.0e5, 1.0e3)
originalExposure.setFilter(originalFilter)
originalExposure.setCalib(originalCalib)
afwWarpedExposure = afwImage.ExposureF(originalExposure.getBBox(),
originalExposure.getWcs())
warpingControl = afwMath.WarpingControl("lanczos4", "", 0,
interpLength)
afwMath.warpExposure(afwWarpedExposure, originalExposure,
warpingControl)
if SAVE_FITS_FILES:
afwWarpedExposure.writeFits("afwWarpedExposureNull.fits")
self.assertEquals(afwWarpedExposure.getFilter().getName(),
originalFilter.getName())
self.assertEquals(afwWarpedExposure.getCalib().getFluxMag0(),
originalCalib.getFluxMag0())
afwWarpedMaskedImage = afwWarpedExposure.getMaskedImage()
afwWarpedMask = afwWarpedMaskedImage.getMask()
edgeBitMask = afwWarpedMask.getPlaneBitMask("EDGE")
if edgeBitMask == 0:
self.fail("warped mask has no EDGE bit")
afwWarpedMaskedImageArrSet = afwWarpedMaskedImage.getArrays()
afwWarpedMaskArr = afwWarpedMaskedImageArrSet[1]
# compare all non-edge pixels of image and variance, but relax specs a bit
# because of minor noise introduced by bad pixels
edgeMaskArr = afwWarpedMaskArr & edgeBitMask
originalMaskedImageArrSet = originalExposure.getMaskedImage(
).getArrays()
errStr = imageTestUtils.maskedImagesDiffer(afwWarpedMaskedImageArrSet,
originalMaskedImageArrSet,
doMask=False,
skipMaskArr=edgeMaskArr,
atol=1e-5)
if errStr:
self.fail(
"afw null-warped MaskedImage (all pixels, relaxed tolerance): %s"
% (errStr, ))
# compare good pixels of image, mask and variance using full tolerance
errStr = imageTestUtils.maskedImagesDiffer(
afwWarpedMaskedImageArrSet,
originalMaskedImageArrSet,
doImage=False,
doVariance=False,
skipMaskArr=afwWarpedMaskArr)
if errStr:
self.fail(
"afw null-warped MaskedImage (good pixels, max tolerance): %s"
% (errStr, ))
# where do you want the data to end up
if os.environ['HOSTNAME'].endswith('ncsa.uiuc.edu'):
# obstest
#BASEDIR = '/lsst/DC3/data/obstest/CFHTLS'
# production link
#BASEDIR = '/usr/data/mysql2/CFHTDeep'
# production actual dir
#BASEDIR = '/usr/data/mysql1/CFHT/CFHTDeep/' # data
BASEDIR = '/usr/data/mysql1/CFHT/' # cals
else:
BASEDIR = None
filterPolicy = pexPolicy.Policy.createPolicy(
os.path.join(eups.productDir("obs_cfht"), "megacam/description/", "MegacamFilters.paf"), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
def saveScience(dim, basedir, fieldid, visitid, filterid, snapid, ccdid, ampid):
# CFHTLS/%(field)/raw/v%(visitid)-f%(filterid)/s%(snapid)/c%(ccdid)-a%(ampid).fits
outdir = '%s/%s/raw/v%s-f%s/s%s' % (basedir, fieldid, visitid, filterid, snapid)
if not os.path.isdir(outdir):
os.makedirs(outdir)
outfile = '%s/c%s-a%s.fits' % (outdir, ccdid, ampid)
print '# writing', outfile
dim.getMetadata().set('DC3BPATH', outfile)
dim.writeFits(outfile)
# where do you want the data to end up
if os.environ['HOSTNAME'].endswith('ncsa.uiuc.edu'):
# obstest
#BASEDIR = '/lsst/DC3/data/obstest/CFHTLS'
# production link
#BASEDIR = '/usr/data/mysql2/CFHTDeep'
# production actual dir
#BASEDIR = '/usr/data/mysql1/CFHT/CFHTDeep/' # data
BASEDIR = '/usr/data/mysql1/CFHT/' # cals
else:
BASEDIR = None
filterPolicy = pexPolicy.Policy.createPolicy(
os.path.join(eups.productDir("obs_cfht"), "megacam/description/",
"MegacamFilters.paf"), True)
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
def saveScience(dim, basedir, fieldid, visitid, filterid, snapid, ccdid,
ampid):
# CFHTLS/%(field)/raw/v%(visitid)-f%(filterid)/s%(snapid)/c%(ccdid)-a%(ampid).fits
outdir = '%s/%s/raw/v%s-f%s/s%s' % (basedir, fieldid, visitid, filterid,
snapid)
if not os.path.isdir(outdir):
os.makedirs(outdir)
outfile = '%s/c%s-a%s.fits' % (outdir, ccdid, ampid)
print '# writing', outfile
dim.getMetadata().set('DC3BPATH', outfile)
