def setUp(self):
defSciencePath = os.path.join(eups.productDir("afwdata"), "DC3a-Sim", "sci", "v26-e0",
"v26-e0-c011-a00.sci")
defTemplatePath = os.path.join(eups.productDir("afwdata"), "DC3a-Sim", "sci", "v5-e0",
"v5-e0-c011-a00.sci")
self.scienceExposure = afwImage.ExposureF(defSciencePath)
self.templateExposure = afwImage.ExposureF(defTemplatePath)
def setUp(self):
#Load sample input from disk
srcSet = readSourceSet(
os.path.join(eups.productDir("meas_pipeline"), "tests",
"v695833-e0-c000.xy.txt"))
exp = afwImg.ExposureF(2048, 4612)
#Put them on the clipboard
fileName = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "WcsDeterminationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(fileName)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("inputExposureKey"), exp)
self.clipboard.put(self.policy.get("inputSourceSetKey"), srcSet)
# Set up local astrometry_net_data
adnDB = 'testTagAlong'
datapath = os.path.join(eups.productDir("meas_pipeline"), 'tests',
'astrometry_net_data', adnDB)
# scons doesn't set $HOME, so make sure it's set
os.environ['HOME'] = os.environ.get("HOME", os.path.expanduser("~"))
eupsObj = eups.Eups(root=datapath)
ok, version, reason = eupsObj.setup('astrometry_net_data')
if not ok:
raise ValueError(
"Need %s version of astrometry_net_data (from path: %s): %s" %
(adnDN, datapath, reason))
def readImage(filename=None):
"""Read an image and background subtract it"""
if not filename:
dataDir = eups.productDir("afwdata")
if not dataDir:
raise RuntimeError("You must provide a filename or setup afwdata to run these examples")
filename = os.path.join(eups.productDir("afwdata"), "CFHT", "D4", "cal-53535-i-797722_1")
bbox = afwGeom.Box2I(afwGeom.Point2I(270, 2530), afwGeom.Extent2I(512, 512))
else:
bbox = None
mi = afwImage.MaskedImageF(filename, 0, None, bbox, afwImage.LOCAL)
mi.setXY0(afwGeom.Point2I(0, 0))
#
# Subtract the background. We'd use a canned procedure, but that's in meas/utils/sourceDetection.py. We
# can't fix those pesky cosmic rays either, as that's in a dependent product (meas/algorithms) too
#
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.NATURAL_SPLINE)
bctrl.setNxSample(int(mi.getWidth()/256) + 1)
bctrl.setNySample(int(mi.getHeight()/256) + 1)
sctrl = bctrl.getStatisticsControl()
sctrl.setNumSigmaClip(3.0)
sctrl.setNumIter(2)
im = mi.getImage()
try:
backobj = afwMath.makeBackground(im, bctrl)
except Exception, e:
print >> sys.stderr, e,
bctrl.setInterpStyle(afwMath.Interpolate.CONSTANT)
backobj = afwMath.makeBackground(im, bctrl)
def testAssembleCcd(self):
"""Test if we can build a Ccd out of Amps"""
#print >> sys.stderr, "Skipping testRotatedCcd"; return
compImage = afwImage.ImageI(os.path.join(eups.productDir("afw"),
"tests", "test_comp.fits.gz"))
compImageTrimmed = afwImage.ImageI(os.path.join(eups.productDir("afw"), "tests",
"test_comp_trimmed.fits.gz"))
ccdId = cameraGeom.Id(1, "LsstLike")
ccdInfo = {"ampSerial" : CameraGeomTestCase.ampSerial}
ccd = cameraGeomUtils.makeCcd(self.geomPolicy, ccdId, ccdInfo=ccdInfo)
#
# Test assembly of images that require preparation for assembly (like
# LSST images)
#
outImage = cameraGeomUtils.makeImageFromCcd(ccd,
imageSource=LsstLikeImage(),
isTrimmed=False, imageFactory=afwImage.ImageU)
self.assertTrue(outImage==compImage)
if display:
cameraGeomUtils.showCcd(ccd, outImage)
ds9.incrDefaultFrame()
ccdId = cameraGeom.Id(1, "ScLike")
ccdInfo = {"ampSerial" : CameraGeomTestCase.ampSerial}
ccd = cameraGeomUtils.makeCcd(self.geomPolicy, ccdId, ccdInfo=ccdInfo)
outImage = cameraGeomUtils.makeImageFromCcd(ccd,
imageSource=ScLikeImage(),
isTrimmed=False, imageFactory=afwImage.ImageU)
self.assertTrue(outImage==compImage)
if display:
cameraGeomUtils.showCcd(ccd, outImage)
ds9.incrDefaultFrame()
#
# Trim the CCD and try again
#
ccd.setTrimmed(True)
ccdId = cameraGeom.Id(1, "LsstLike Trimmed")
ccdInfo = {"ampSerial" : CameraGeomTestCase.ampSerial}
ccd = cameraGeomUtils.makeCcd(self.geomPolicy, ccdId, ccdInfo=ccdInfo)
outImage = cameraGeomUtils.makeImageFromCcd(ccd,
imageSource=LsstLikeImage(),
isTrimmed=True, imageFactory=afwImage.ImageU)
self.assertTrue(outImage==compImage)
if display:
cameraGeomUtils.showCcd(ccd, outImage)
ds9.incrDefaultFrame()
def setUp(self):
self.FWHM = 5
self.psf = algorithms.DoubleGaussianPsf(15, 15, self.FWHM/(2*sqrt(2*log(2))))
maskedImageFile = os.path.join(eups.productDir("afwdata"), "CFHT", "D4", "cal-53535-i-797722_1.fits")
self.mi = afwImage.MaskedImageF(maskedImageFile)
if False: # use sub-image?
self.mi = self.mi.Factory(self.mi, afwImage.BBox(afwImage.PointI(760, 20), 256, 256))
self.mi.getMask().addMaskPlane("INTERP")
self.badPixels = defects.policyToBadRegionList(os.path.join(eups.productDir("meas_algorithms"),
"policy", "BadPixels.paf"))
def setUp(self):
filename = os.path.join(eups.productDir("afwdata"), "CFHT", "D4",
"cal-53535-i-797722_1")
bbox = afwGeom.Box2I(afwGeom.Point2I(32, 32),
afwGeom.Extent2I(1024, 1024))
self.exposure = afwImage.ExposureF(filename, 0, bbox, afwImage.LOCAL)
def testCFHT_oldAPI(self):
"""Test background subtraction on some real CFHT data"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testCFHT as afwdata is not setup"
return
mi = afwImage.MaskedImageF(
os.path.join(afwdataDir, "CFHT", "D4",
"cal-53535-i-797722_1.fits"))
mi = mi.Factory(
mi,
afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)),
afwImage.LOCAL)
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.AKIMA_SPLINE)
bctrl.setNxSample(16)
bctrl.setNySample(16)
bctrl.getStatisticsControl().setNumSigmaClip(3.0)
bctrl.getStatisticsControl().setNumIter(2)
backobj = afwMath.makeBackground(mi.getImage(), bctrl)
if display:
ds9.mtv(mi, frame=0)
im = mi.getImage()
im -= backobj.getImageF()
if display:
ds9.mtv(mi, frame=1)
def testTicket987(self):
"""This code used to abort; so the test is that it doesn't"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testTicket987 as afwdata is not setup"
return
imagePath = os.path.join(afwdataDir, "DC3a-Sim", "sci", "v5-e0",
"v5-e0-c011-a00.sci.fits")
mimg = afwImage.MaskedImageF(imagePath)
binsize = 512
bctrl = afwMath.BackgroundControl("NATURAL_SPLINE")
### Adding this line solves the problem ###
# note: by default undersampleStyle is THROW_EXCEPTION
bctrl.setUndersampleStyle(afwMath.REDUCE_INTERP_ORDER)
################################################
nx = int(mimg.getWidth() / binsize) + 1
ny = int(mimg.getHeight() / binsize) + 1
#print 'Binning', nx, ny
bctrl.setNxSample(nx)
bctrl.setNySample(ny)
image = mimg.getImage()
backobj = afwMath.makeBackground(image, bctrl)
image -= backobj.getImageF()
def testSingleExposure(self):
policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "CrRejectStageDictionary.paf", "policy")
defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True)
policy = pexPolicy.Policy(os.path.join(eups.productDir("ip_pipeline"),
"tests", "crReject_policy.paf"))
policy.mergeDefaults(defPolicy)
stage = ipPipe.CrRejectStage(policy)
tester = SimpleStageTester(stage)
clipboard = pexClipboard.Clipboard()
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
if display:
ds9.mtv(self.exposure, frame=0, title="Input")
#
# Do the work
#
outWorker = tester.runWorker(clipboard)
outPolicy = policy.get("outputKeys")
self.assertTrue(outWorker.contains(outPolicy.get("exposure")))
self.assertEqual(outWorker.get("nCR"), 25)
if display:
ds9.mtv(outWorker.get(outPolicy.get("exposure")), frame=1, title="CR removed")
def setUp(self):
random.seed(1369)
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different positions
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
catSet = []
for s in srcSet:
s1 = afwDet.Source(s)
s1.setXAstrom( s1.getXAstrom()+ random.uniform(-.1,.1))
catSet.append(s1)
#Make a SourceMatch object
maxDist = 1/3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile("meas_pipeline",
"WcsVerificationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(4000, 4000)))
def suite():
"""Returns a suite containing all the test cases in this module."""
utilsTests.init()
suites = []
if not eups.productDir("afwdata"):
print >> sys.stderr, "afwdata is not setting up; skipping test"
elif not eups.productDir("obs_lsstSim"):
print >> sys.stderr, "obs_lsstSim not set up; skipping test"
else:
suites += unittest.makeSuite(IsrCcdAssemblyTestCase)
suites += unittest.makeSuite(utilsTests.MemoryTestCase)
return unittest.TestSuite(suites)
def setUp(self):
self.Mask = afwImage.MaskU
self.Mask.clearMaskPlaneDict() # reset so tests will be deterministic
for p in ("BAD", "SAT", "INTRP", "CR", "EDGE"):
self.Mask.addMaskPlane(p)
self.BAD = self.Mask.getPlaneBitMask("BAD")
self.CR = self.Mask.getPlaneBitMask("CR")
self.EDGE = self.Mask.getPlaneBitMask("EDGE")
self.val1 = self.BAD | self.CR
self.val2 = self.val1 | self.EDGE
self.mask1 = afwImage.MaskU(100, 200)
self.mask1.set(self.val1)
self.mask2 = afwImage.MaskU(self.mask1.getDimensions())
self.mask2.set(self.val2)
dataDir = os.path.join(eups.productDir("afwdata"), "data")
if dataDir:
if True:
self.maskFile = os.path.join(dataDir, "small_MI_msk.fits")
else:
self.maskFile = os.path.join(dataDir, "871034p_1_MI_msk.fits")
else:
self.maskFile = None
def __init__(self, dbm, database):
self.dbm = dbm
submissionsTableName = "submissions"
totalsTableName = "totals"
updatesTableName = "updates"
#
# This load the submissions.sql, which creates the table
# we're writing into. The table won't be created
# if it already exists. (see the SQL for details).
pkg = eups.productDir("ctrl_stats")
filePath = os.path.join(pkg, "sql", "eventCodes.sql")
dbm.loadSql(filePath, database)
filePath = os.path.join(pkg, "sql", "submissions.sql")
dbm.loadSql(filePath, database)
filePath = os.path.join(pkg, "sql", "totals.sql")
dbm.loadSql(filePath, database)
filePath = os.path.join(pkg, "sql", "updates.sql")
dbm.loadSql(filePath, database)
# full name of the submissions table
self.submissionsTable = database + "." + submissionsTableName
# full name of the updates table
self.updatesTable = database + "." + updatesTableName
# full name of the totals table
self.totalsTable = database + "." + totalsTableName
def testCFHT_oldAPI(self):
"""Test background subtraction on some real CFHT data"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testCFHT as afwdata is not setup"
return
mi = afwImage.MaskedImageF(os.path.join(afwdataDir,
"CFHT", "D4", "cal-53535-i-797722_1"))
mi = mi.Factory(mi, afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)), afwImage.LOCAL)
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.AKIMA_SPLINE)
bctrl.setNxSample(16)
bctrl.setNySample(16)
bctrl.getStatisticsControl().setNumSigmaClip(3.0)
bctrl.getStatisticsControl().setNumIter(2)
backobj = afwMath.makeBackground(mi.getImage(), bctrl)
if display:
ds9.mtv(mi, frame = 0)
im = mi.getImage()
im -= backobj.getImageF()
if display:
ds9.mtv(mi, frame = 1)
def setUp(self):
self.qaTask = PipeQaTask()
self.testDatabase = "abecker_pipeQA_unittest" # lsstSim S21 schema
self.testVisit1 = "899551571" # z-band
self.testVisit2 = "899553091" # r-band
self.testFilt1 = "z"
self.testFilt2 = "r"
self.testRaft = "2,2"
self.testCcd = "1,1"
os.environ["WWW_ROOT"] = os.path.join(
eups.productDir("testing_pipeQA"), "tests")
self.wwwRoot = os.environ["WWW_ROOT"]
self.wwwRerun = "www_rerun"
os.environ["WWW_RERUN"] = self.wwwRerun
if os.path.isdir(self.wwwRerun):
shutil.rmtree(self.wwwRerun)
self.wwwPath = os.path.join(self.wwwRoot, self.wwwRerun)
# Test if database access is enabled; if not the tests will not be run
self.run = True
self.log = pexLog.Log(pexLog.Log.getDefaultLog(),
'lsst.testing.pipeQA.PipeQaTestCases',
pexLog.Log.INFO)
try:
dbid = DatabaseIdentity(self.testDatabase)
except pexExcept.LsstCppException, e:
self.log.warn("Unable to create database identity: %s" % e.message)
self.run = False
return
def suite():
"""Returns a suite containing all the test cases in this module."""
utilsTests.init()
suites = []
if not eups.productDir("afwdata"):
print >> sys.stderr, "afwdata is not setting up; skipping test"
elif not eups.productDir("obs_lsstSim"):
print >> sys.stderr, "obs_lsstSim not set up; skipping test"
else:
suites += unittest.makeSuite(IsrPipelineTestCase)
suites += unittest.makeSuite(utilsTests.MemoryTestCase)
return unittest.TestSuite(suites)
def testCFHT(self):
"""Test background subtraction on some real CFHT data"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testCFHT as afwdata is not setup"
return
mi = afwImage.MaskedImageF(os.path.join(afwdataDir,
"CFHT", "D4", "cal-53535-i-797722_1"))
mi = mi.Factory(mi, afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)), afwImage.LOCAL)
bctrl = afwMath.BackgroundControl(mi.getWidth()//128, mi.getHeight()//128)
bctrl.getStatisticsControl().setNumSigmaClip(3.0)
bctrl.getStatisticsControl().setNumIter(2)
backobj = afwMath.makeBackground(mi.getImage(), bctrl)
if display:
ds9.mtv(mi, frame = 0)
im = mi.getImage()
im -= backobj.getImageF("AKIMA_SPLINE")
if display:
ds9.mtv(mi, frame = 1)
statsImage = afwMath.cast_BackgroundMI(backobj).getStatsImage()
if display:
ds9.mtv(backobj.getStatsImage(), frame=2)
ds9.mtv(backobj.getStatsImage().getVariance(), frame=3)
def setUp(self):
self.Mask = afwImage.MaskU
self.Mask.clearMaskPlaneDict() # reset so tests will be deterministic
for p in ("BAD", "SAT", "INTRP", "CR", "EDGE"):
self.Mask.addMaskPlane(p)
self.BAD = self.Mask.getPlaneBitMask("BAD")
self.CR = self.Mask.getPlaneBitMask("CR")
self.EDGE = self.Mask.getPlaneBitMask("EDGE")
self.val1 = self.BAD | self.CR
self.val2 = self.val1 | self.EDGE
self.mask1 = afwImage.MaskU(100, 200)
self.mask1.set(self.val1)
self.mask2 = afwImage.MaskU(self.mask1.getDimensions())
self.mask2.set(self.val2)
dataDir = os.path.join(eups.productDir("afwdata"), "data")
if dataDir:
if True:
self.maskFile = os.path.join(dataDir, "small_MI.fits")
else:
self.maskFile = os.path.join(dataDir, "871034p_1_MI.fits")
else:
self.maskFile = None
def __init__(self, dbm, database):
self.dbm = dbm
submissionsTableName = "submissions"
totalsTableName = "totals"
updatesTableName = "updates"
#
# This load the submissions.sql, which creates the table
# we're writing into. The table won't be created
# if it already exists. (see the SQL for details).
pkg = eups.productDir("ctrl_stats")
filePath = os.path.join(pkg, "sql", "eventCodes.sql")
dbm.loadSql(filePath, database)
filePath = os.path.join(pkg, "sql", "submissions.sql")
dbm.loadSql(filePath, database)
filePath = os.path.join(pkg, "sql", "totals.sql")
dbm.loadSql(filePath, database)
filePath = os.path.join(pkg, "sql", "updates.sql")
dbm.loadSql(filePath, database)
# full name of the submissions table
self.submissionsTable = database + "." + submissionsTableName
# full name of the updates table
self.updatesTable = database + "." + updatesTableName
# full name of the totals table
self.totalsTable = database + "." + totalsTableName
def setUp(self):
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different fluxes.
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
#Also need to specify that each source is a star
catSet = []
flags = malgUtil.getDetectionFlags()
goodFlag = flags['BINNED1'] | flags['STAR']
for s in srcSet:
s1 = afwDet.Source(s)
s1.setPsfFlux(s1.getPsfFlux() * .281)
catSet.append(s1)
s.setFlagForDetection(goodFlag)
#Make a SourceMatch object
maxDist = 1 / 3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile("meas_pipeline",
"PhotoCalStageDictionary.paf",
"policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(10, 10)))
def testReadFits(self):
"""Test reading FITS files"""
dataDir = eups.productDir("afwdata")
if dataDir:
dataDir = os.path.join(dataDir, "data")
else:
print >> sys.stderr, "Warning: afwdata is not set up; not running the FITS I/O tests"
return
hdus = {}
fileName = os.path.join(dataDir, "small_MI.fits")
hdus["img"] = 2 # an S16 fits HDU
hdus["msk"] = 3 # an U8 fits HDU
hdus["var"] = 4 # an F32 fits HDU
imgU = afwImage.DecoratedImageU(fileName, hdus["img"]) # read as unsigned short
imgF = afwImage.DecoratedImageF(fileName, hdus["img"]) # read as float
self.assertEqual(imgU.getHeight(), 256)
self.assertEqual(imgF.getImage().getWidth(), 256)
self.assertEqual(imgU.getImage().get(0, 0), imgF.getImage().get(0, 0))
#
# Check the metadata
#
meta = self.trueMetadata
for k in meta.keys():
self.assertEqual(imgU.getMetadata().getAsDouble(k), meta[k])
self.assertEqual(imgF.getMetadata().getAsDouble(k), meta[k])
#
# Read an F32 image
#
varU = afwImage.DecoratedImageF(fileName, hdus["var"]) # read as unsigned short
varF = afwImage.DecoratedImageF(fileName, hdus["var"]) # read as float
self.assertEqual(varU.getHeight(), 256)
self.assertEqual(varF.getImage().getWidth(), 256)
self.assertEqual(varU.getImage().get(0, 0), varF.getImage().get(0, 0))
#
# Read a char image
#
maskImg = afwImage.DecoratedImageU(fileName, hdus["msk"]).getImage() # read a char file
self.assertEqual(maskImg.getHeight(), 256)
self.assertEqual(maskImg.getWidth(), 256)
self.assertEqual(maskImg.get(0, 0), 1)
#
# Read a U16 image
#
tmpFile = "foo.fits"
imgU.writeFits(tmpFile)
try:
imgU16 = afwImage.DecoratedImageF(tmpFile) # read as unsigned short
except:
os.remove(tmpFile)
raise
os.remove(tmpFile)
def ConfigureDependentProducts(productName, dependencyFilename="dependencies.dat"):
"""Process a product's dependency file, returning a list suitable for passing to SconsUtils.makeEnv"""
productDir = eups.productDir(productName)
if not productDir:
raise RuntimeError, ("%s is not setup" % productName)
dependencies = os.path.join(productDir, "etc", dependencyFilename)
try:
fd = open(dependencies)
except:
raise RuntimeError, ("Unable to lookup dependencies for %s" % productName)
dependencies = []
for line in fd.readlines():
if re.search(r"^\s*#", line):
continue
mat = re.search(r"^(\S+):\s*$", line)
if mat:
dependencies += ConfigureDependentProducts(mat.group(1))
continue
#
# Split the line into "" separated fields
#
line = re.sub(r"(^\s*|\s*,\s*|\s*$)", "", line) # remove whitespace and commas in the config file
dependencies.append([f for f in re.split(r"['\"]", line) if f])
return dependencies
def setUp(self):
random.seed(1369)
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different positions
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
catSet = []
for s in srcSet:
s1 = afwDet.Source(s)
s1.setXAstrom(s1.getXAstrom() + random.uniform(-.1, .1))
catSet.append(s1)
#Make a SourceMatch object
maxDist = 1 / 3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "WcsVerificationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(4000, 4000)))
def testSingleExposure(self):
policyFile = pexPolicy.DefaultPolicyFile(
"ip_pipeline", "CrRejectStageDictionary.paf", "policy")
defPolicy = pexPolicy.Policy.createPolicy(
policyFile, policyFile.getRepositoryPath(), True)
policy = pexPolicy.Policy(
os.path.join(eups.productDir("ip_pipeline"), "tests",
"crReject_policy.paf"))
policy.mergeDefaults(defPolicy)
stage = ipPipe.CrRejectStage(policy)
tester = SimpleStageTester(stage)
clipboard = pexClipboard.Clipboard()
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
if display:
ds9.mtv(self.exposure, frame=0, title="Input")
#
# Do the work
#
outWorker = tester.runWorker(clipboard)
outPolicy = policy.get("outputKeys")
self.assertTrue(outWorker.contains(outPolicy.get("exposure")))
self.assertEqual(outWorker.get("nCR"), 25)
if display:
ds9.mtv(outWorker.get(outPolicy.get("exposure")),
frame=1,
title="CR removed")
def setUp(self):
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different fluxes.
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
#Also need to specify that each source is a star
catSet = []
flags = malgUtil.getDetectionFlags()
goodFlag = flags['BINNED1'] | flags['STAR']
for s in srcSet:
s1 = afwDet.Source(s)
s1.setPsfFlux(s1.getPsfFlux()*.281)
catSet.append(s1)
s.setFlagForDetection(goodFlag)
#Make a SourceMatch object
maxDist = 1/3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile("meas_pipeline",
"PhotoCalStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"), afwImage.ExposureF(afwGeom.Extent2I(10, 10)))
def testTicket987(self):
"""This code used to abort; so the test is that it doesn't"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testTicket987 as afwdata is not setup"
return
imagePath = os.path.join(afwdataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci")
mimg = afwImage.MaskedImageF(imagePath)
binsize = 512
bctrl = afwMath.BackgroundControl("NATURAL_SPLINE")
### Adding this line solves the problem ###
# note: by default undersampleStyle is THROW_EXCEPTION
bctrl.setUndersampleStyle(afwMath.REDUCE_INTERP_ORDER)
################################################
nx = int(mimg.getWidth()/binsize) + 1
ny = int(mimg.getHeight()/binsize) + 1
#print 'Binning', nx, ny
bctrl.setNxSample(nx)
bctrl.setNySample(ny)
image = mimg.getImage()
backobj = afwMath.makeBackground(image, bctrl)
image -= backobj.getImageF()
def testBackgroundTestImages(self):
imginfolist = []
#imginfolist.append( ["v1_i1_g_m400_s20_f.fits", 400.05551471441612] ) # cooked to known value
#imginfolist.append( ["v1_i1_g_m400_s20_f.fits", 400.00295902395123] ) # cooked to known value
#imginfolist.append( ["v1_i1_g_m400_s20_f.fits", 400.08468385712251] ) # cooked to known value
#imginfolist.append( ["v1_i1_g_m400_s20_f.fits", 400.00305806663295] ) # cooked to known value
#imginfolist.append( ["v1_i1_g_m400_s20_f.fits", 400.0035102188698] ) # cooked to known value
imginfolist.append( ["v1_i1_g_m400_s20_f.fits", 399.9912966583894] ) # cooked to known value
#imgfiles.append("v1_i1_g_m400_s20_u16.fits")
#imgfiles.append("v1_i2_g_m400_s20_f.fits"
#imgfiles.append("v1_i2_g_m400_s20_u16.fits")
#imgfiles.append("v2_i1_p_m9_f.fits")
#imgfiles.append("v2_i1_p_m9_u16.fits")
#imgfiles.append("v2_i2_p_m9_f.fits")
#imgfiles.append("v2_i2_p_m9_u16.fits")
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testBackgroundTestImages as afwdata is not setup"
return
for imginfo in imginfolist:
imgfile, centerValue = imginfo
imgPath = afwdataDir + "/Statistics/" + imgfile
# get the image and header
dimg = afwImage.DecoratedImageD(imgPath)
img = dimg.getImage()
fitsHdr = dimg.getMetadata() # the FITS header
# get the True values of the mean and stdev
reqMean = fitsHdr.getAsDouble("MEANREQ")
reqStdev = fitsHdr.getAsDouble("SIGREQ")
naxis1 = img.getWidth()
naxis2 = img.getHeight()
# create a background control object
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.AKIMA_SPLINE)
bctrl.setNxSample(5)
bctrl.setNySample(5)
# run the background constructor and call the getPixel() and getImage() functions.
backobj = afwMath.makeBackground(img, bctrl)
pixPerSubimage = img.getWidth()*img.getHeight()/(bctrl.getNxSample()*bctrl.getNySample())
stdevInterp = reqStdev/math.sqrt(pixPerSubimage)
# test getPixel()
testval = afwMath.cast_BackgroundMI(backobj).getPixel(naxis1/2, naxis2/2)
self.assertAlmostEqual(testval/centerValue, 1, places=7)
self.assertTrue( abs(testval - reqMean) < 2*stdevInterp )
# test getImage() by checking the center pixel
bimg = backobj.getImageD()
testImgval = bimg.get(naxis1/2, naxis2/2)
self.assertTrue( abs(testImgval - reqMean) < 2*stdevInterp )
def testProductDir(self):
if "PYTHON_DIR" in os.environ:
del os.environ["PYTHON_DIR"]
if "SETUP_PYTHON" in os.environ:
del os.environ["SETUP_PYTHON"]
self.assertTrue(
eups.productDir("python") is None, "found unsetup product")
pdir = os.path.join(testEupsStack, os.environ["EUPS_FLAVOR"], "python",
"2.5.2")
dir = eups.productDir("python", "2.5.2")
self.assertEqual(dir, pdir)
eups.setup("python", "2.5.2")
dir = eups.productDir("python")
self.assertEqual(dir, pdir)
def getImage(self, ccd, amp, imageFactory=afwImage.ImageU):
im = imageFactory(os.path.join(eups.productDir("afw"), "tests",
"test.fits.gz"))
if self.isTrimmed:
bbox = amp.getDataSec()
else:
bbox = amp.getAllPixels()
return imageFactory(im, bbox, afwImage.LOCAL)
def getExamples(self, filename=None):
if not self.examplesDir:
self.examplesDir = os.path.join(eups.productDir("pex_policy"),
"examples")
if filename:
return os.path.join(self.examplesDir, filename)
else:
return self.examplesDir
def testProductDir(self):
if "PYTHON_DIR" in os.environ:
del os.environ["PYTHON_DIR"]
if "SETUP_PYTHON" in os.environ:
del os.environ["SETUP_PYTHON"]
self.assert_(eups.productDir("python") is None,
"found unsetup product")
pdir = os.path.join(testEupsStack, os.environ["EUPS_FLAVOR"],
"python", "2.5.2")
dir = eups.productDir("python", "2.5.2")
self.assertEquals(dir, pdir)
eups.setup("python", "2.5.2")
dir = eups.productDir("python")
self.assertEquals(dir, pdir)
def testMakeWcs(self):
path = eups.productDir("afw")
path = os.path.join(path, "tests", "data", "parent.fits")
fitsHdr = image.readMetadata(path)
wcs = image.makeWcs(fitsHdr)
c = wcs.pixelToSky(0, 0)
print type(c)
c.getPosition()
def testMakeWcs(self):
path= eups.productDir("afw")
path = os.path.join(path, "tests", "data", "parent.fits")
fitsHdr = image.readMetadata(path)
wcs = image.makeWcs(fitsHdr)
c = wcs.pixelToSky(0,0)
print type(c)
c.getPosition()
def suite():
"""Returns a suite containing all the test cases in this module."""
tests.init()
suites = []
if eups.productDir("afwdata"):
suites += unittest.makeSuite(interpolationTestCase)
else:
print "Skipping interpolation test case as afwdata isn't set up"
suites += unittest.makeSuite(tests.MemoryTestCase)
return unittest.TestSuite(suites)
def testTicket653(self):
"""How-to-repeat for #653"""
# The original ticket read this file, but it doesn't reproduce for me,
# As I don't see how reading an exposure from disk could make a difference
# it's easier to just build an Image
if False:
im = afwImage.ImageF(os.path.join(eups.productDir("afwdata"), "med_img.fits"))
else:
im = afwImage.ImageF(afwGeom.Extent2I(10, 10))
mi = afwImage.MaskedImageF(im)
exp = afwImage.ExposureF(mi)
def testTicket653(self):
"""How-to-repeat for #653"""
# The original ticket read this file, but it doesn't reproduce for me,
# As I don't see how reading an exposure from disk could make a difference
# it's easier to just build an Image
if False:
im = afwImage.ImageF(
os.path.join(eups.productDir("afwdata"), "med_img.fits"))
else:
im = afwImage.ImageF(afwGeom.Extent2I(10, 10))
mi = afwImage.MaskedImageF(im)
exp = afwImage.ExposureF(mi)
def suite():
"""Returns a suite containing all the test cases in this module."""
utilsTests.init()
suites = []
if not eups.productDir("afwdata"):
print >> sys.stderr, "afwdata is not setting up; skipping test"
else:
suites += unittest.makeSuite(PsfDeterminationStageTestCase)
suites += unittest.makeSuite(utilsTests.MemoryTestCase)
return unittest.TestSuite(suites)
def setUp(self):
# Initialise our filters
#
# Start by forgetting that we may already have defined filters
#
filterPolicy = pexPolicy.Policy.createPolicy(
os.path.join(eups.productDir("afw"), "tests", "SdssFilters.paf"), True)
self.filters = tuple(sorted([f.get("name") for f in filterPolicy.getArray("Filter")]))
imageUtils.defineFiltersFromPolicy(filterPolicy, reset=True)
self.g_lambdaEff = [p.get("lambdaEff") for p in filterPolicy.getArray("Filter")
if p.get("name") == "g"][0] # used for tests
def suite():
"""Returns a suite containing all the test cases in this module."""
tests.init()
suites = []
suites += unittest.makeSuite(MeasureTestCase)
if eups.productDir("afwdata"):
suites += unittest.makeSuite(FindAndMeasureTestCase)
else:
print >> sys.stderr, "You must set up afwdata to run the CFHT-based tests"
suites += unittest.makeSuite(GaussianPsfTestCase)
suites += unittest.makeSuite(tests.MemoryTestCase)
return unittest.TestSuite(suites)
def setUp(self):
self.val1, self.val2 = 10, 100
self.width, self.height = 200, 100
self.dimage1 = afwImage.DecoratedImageF(
afwGeom.Extent2I(self.width, self.height)
)
self.dimage1.getImage().set(self.val1)
dataDir = eups.productDir("afwdata")
if dataDir:
self.fileForMetadata = os.path.join(dataDir, "data", "small_MI_img.fits")
self.trueMetadata = {"RELHUMID" : 10.69}
else:
self.fileForMetadata = None
def suite():
"""Returns a suite containing all the test cases in this module."""
utilsTests.init()
suites = []
if not eups.productDir("astrometry_net_data"):
print >> sys.stderr, "Unable to test WcsDeterminationStage as astrometry_net_data is not setup"
else:
suites += unittest.makeSuite(PhotoCalStageTestCase)
suites += unittest.makeSuite(utilsTests.MemoryTestCase)
return unittest.TestSuite(suites)
def suite():
"""Returns a suite containing all the test cases in this module."""
utilsTests.init()
suites = []
if not eups.productDir("afwdata"):
print >> sys.stderr, "afwdata is not setting up; skipping test"
else:
suites += unittest.makeSuite(BackgroundEstimationStageTestCase)
suites += unittest.makeSuite(utilsTests.MemoryTestCase)
return unittest.TestSuite(suites)
def suite():
"""Returns a suite containing all the test cases in this module."""
utilsTests.init()
suites = []
if not eups.productDir("astrometry_net_data"):
print >> sys.stderr, "Unable to test WcsDeterminationStage as astrometry_net_data is not setup"
else:
suites += unittest.makeSuite(PhotoCalStageTestCase)
suites += unittest.makeSuite(utilsTests.MemoryTestCase)
return unittest.TestSuite(suites)
def setUp(self):
self.val1, self.val2 = 10, 100
self.width, self.height = 200, 100
self.dimage1 = afwImage.DecoratedImageF(
afwGeom.Extent2I(self.width, self.height)
)
self.dimage1.getImage().set(self.val1)
dataDir = eups.productDir("afwdata")
if dataDir:
self.fileForMetadata = os.path.join(dataDir, "data", "small_MI.fits")
self.trueMetadata = {"RELHUMID" : 10.69}
else:
self.fileForMetadata = None
def setUp(self):
eupsObj = eups.Eups()
#ver = 'imsim-2011-08-01-0'
# ver = 'sdss-2012-05-01-0'
# ok, version, reason = eupsObj.setup('astrometry_net_data', versionName=ver)
# if not ok:
# raise ValueError("Failed to setup astrometry_net_data version '%s': %s" % (ver, reason))
# print 'Setup astrometry_net_data', ver
mypath = eups.productDir("pipe_tasks")
datapath = os.path.join(mypath, 'tests', 'astrometry_net_data', 't2155')
eupsObj = eups.Eups(root=datapath)
ok, version, reason = eupsObj.setup('astrometry_net_data')
if not ok:
raise ValueError("Need local version of astrometry_net_data (from path: %s): %s" %
(datapath, reason))
def setUp(self):
afwDataDir = eups.productDir("afwdata")
self.root = afwDataDir+"/ImSim"
if not os.path.exists("registry.sqlite3"):
os.symlink(os.path.join(self.root, "registry.sqlite3"),
"./registry.sqlite3")
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put('jobIdentity', {
'visit': 85751839, 'snap': 0,
'raft': "2,3", 'sensor': "1,1", 'channel': "0,0", 'filter': "r"
})
self.clipboard.put('inputDatasets', [
Dataset("raw", visit=85751839, snap=0,
raft="2,3", sensor="1,1", channel="0,0", filter="r")
])
def run(exit=False):
"""Run the tests"""
if False:
import eups
dataDir = eups.productDir("afwdata")
if not dataDir:
raise RuntimeError("Must set up afwdata to run me")
im = afwImage.ImageF(os.path.join(dataDir, "871034p_1_img.fits"))
else:
im = afwImage.ImageF(afwGeom.Extent2I(256, 256))
addCosmicRays(im, nCR=100, emin=800, emax=1000)
ds9.mtv(im)
def setUp(self):
self.mi = afwImage.MaskedImageF(os.path.join(eups.productDir("afwdata"),
"CFHT", "D4", "cal-53535-i-797722_1.fits"))
self.FWHM = 5
self.psf = algorithms.DoubleGaussianPsf(15, 15, self.FWHM/(2*sqrt(2*log(2))))
if False: # use full image, trimmed to data section
self.XY0 = afwGeom.PointI(32, 2)
self.mi = self.mi.Factory(self.mi, afwGeom.BoxI(self.XY0, afwGeom.PointI(2079, 4609)), afwImage.LOCAL)
self.mi.setXY0(afwGeom.PointI(0, 0))
else: # use sub-image
self.XY0 = afwGeom.PointI(824, 140)
self.mi = self.mi.Factory(self.mi, afwGeom.BoxI(self.XY0, afwGeom.ExtentI(256, 256)), afwImage.LOCAL)
self.mi.getMask().addMaskPlane("DETECTED")
self.exposure = afwImage.makeExposure(self.mi)
def readData(self, fileName = None, subImage = False):
afwdataDir = eups.productDir("afwdata") or ""
if not fileName or isinstance(fileName, int):
if fileName:
which = fileName
else:
which = 1
fileName = os.path.join(afwdataDir, "med")
elif not os.path.exists(fileName):
fileName = os.path.join(afwdataDir, fileName)
#
# We could read into an Exposure, but we're going to want to determine our own WCS
#
hdu, metadata = 0, dafBase.PropertySet()
if True:
mi = afwImage.MaskedImageF(fileName, hdu, metadata) # read MaskedImage
self.XY0 = mi.getXY0()
else:
if subImage: # use sub-image
self.XY0 = afwImage.PointI(824, 140)
bbox = afwImage.BBox(self.XY0, 512, 512)
else: # use full image, trimmed to data section
self.XY0 = afwImage.PointI(32, 2)
bbox = afwImage.BBox(self.XY0, afwImage.PointI(2079, 4609))
mi = afwImage.MaskedImageF(fileName, hdu, metadata, bbox) # read MaskedImage
if not subImage:
mi.setXY0(afwImage.PointI(0, 0)) # we just trimmed the overscan
wcs = afwImage.makeWcs(metadata)
self.pixscale = wcs.pixelScale().asArcseconds()
#
# Just an initial guess
#
FWHM = 5
self.psf = measAlg.DoubleGaussianPsf(15, 15, FWHM/(2*sqrt(2*log(2))))
mi.getMask().addMaskPlane("DETECTED")
self.exposure = afwImage.makeExposure(mi, wcs)
self.exposure.setPsf(self.psf)
if self.display:
ds9.mtv(self.exposure)
def setUp(self):
self.sources = afwDet.SourceSet()
for i in xrange(1, 10):
s = afwDet.Source()
s.setXFlux(i * 10.0)
s.setXFluxErr(i * 0.01)
s.setYFlux(i * 10.0)
s.setYFluxErr(i * 0.01)
s.setXAstrom(i * 10.0)
s.setXAstromErr(i * 0.01)
s.setYAstrom(i * 10.0)
s.setYAstromErr(i * 0.01)
s.setXPeak(i * 10.0)
s.setYPeak(i * 10.0)
self.sources.append(s)
imagePath = os.path.join(eups.productDir("meas_pipeline"), "tests",
"v695833-e0-c000-a00.sci_img.fits")
self.wcs = afwImage.makeWcs(afwImage.readMetadata(imagePath))
def clipboardIoTest(self):
file = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "tests/sourceDetection0_policy.paf")
detectPolicy = pexPolicy.Policy.createPolicy(file)
file = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "tests/sourceMeasurement0_policy.paf")
measurePolicy = pexPolicy.Policy.createPolicy(file)
file = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "tests/psfDetermination0_policy.paf")
psfPolicy = pexPolicy.Policy.createPolicy(file)
tester = SimpleStageTester()
tester.addStage(measPipe.SourceDetectionStage(detectPolicy))
tester.addStage(measPipe.SourceMeasurementStage(measurePolicy))
tester.addStage(measPipe.PsfDeterminationStage(psfPolicy))
clipboard = pexClipboard.Clipboard()
filename = os.path.join(eups.productDir("afwdata"), "CFHT", "D4",
"cal-53535-i-797722_1")
# test only a portion of the exposure to speed up testing
bbox = afwGeom.Box2I(afwGeom.Point2I(32, 32),
afwGeom.Extent2I(512, 512))
testExp = afwImage.ExposureF(filename, 0, bbox, afwImage.LOCAL)
# test full image
# testExp = afImage.ExposureF(filename)
clipboard = pexClipboard.Clipboard()
clipboard.put(detectPolicy.get("inputKeys.exposure"), testExp)
outWorker = tester.runWorker(clipboard)
assert (outWorker.contains(psfPolicy.get("data.outputPsfKey")))
assert (outWorker.contains(psfPolicy.get("data.outputCellSetKey")))
del outWorker
del testExp
del clipboard
del tester
print >> sys.stderr, "at end of test"
def loadData():
"""Prepare the data we need to run the example"""
# Load sample input from disk
mypath = eups.productDir("afwdata")
if not mypath:
print >> sys.stderr, "Please setup afwdata and try again"
sys.exit(1)
imFile = os.path.join(mypath, "CFHT", "D4",
"cal-53535-i-797722_small_1.fits")
exposure = afwImage.ExposureF(imFile)
psf = measAlg.SingleGaussianPsf(21, 21, 2)
exposure.setPsf(psf)
im = exposure.getMaskedImage().getImage()
im -= float(numpy.median(im.getArray()))
return exposure
def setUp(self):
path = eups.productDir("afw")
self.parentFile = os.path.join(path, "tests", "data", "parent.fits")
self.parent = afwImage.ExposureF(self.parentFile)
self.llcParent = self.parent.getMaskedImage().getXY0()
self.oParent = self.parent.getWcs().getPixelOrigin()
#A list of pixel positions to test
self.testPositions = []
self.testPositions.append(afwGeom.Point2D(128, 128))
self.testPositions.append(afwGeom.Point2D(0, 0))
self.testPositions.append(afwGeom.Point2D(20, 30))
self.testPositions.append(afwGeom.Point2D(60, 50))
self.testPositions.append(afwGeom.Point2D(80, 80))
self.testPositions.append(afwGeom.Point2D(255, 255))
self.parent.getMaskedImage().set(0)
for p in self.testPositions:
self.parent.getMaskedImage().set(int(p[0]), int(p[1]),
(10 + p[0], ))
def testDistortionInACamera(self):
policyFile = cameraGeomUtils.getGeomPolicy(
os.path.join(eups.productDir("afw"), "tests",
"TestCameraGeom.paf"))
pol = pexPolicy.Policy(policyFile)
pol = cameraGeomUtils.getGeomPolicy(pol)
cam = cameraGeomUtils.makeCamera(pol)
# see if the distortion object made it into the camera object
dist = cam.getDistortion()
self.tryAFewCoords(dist, [1.0, 1.0, 0.1])
# see if the distortion object is accessible in the ccds
for raft in cam:
for ccd in cameraGeom.cast_Raft(raft):
ccd = cameraGeom.cast_Ccd(ccd)
if self.prynt:
print "CCD id: ", ccd.getId()
ccdDist = ccd.getDistortion()
self.tryAFewCoords(dist, [1.0, 1.0, 0.1])
