def setUp(self):
random.seed(1369)
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different positions
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
catSet = []
for s in srcSet:
s1 = afwDet.Source(s)
s1.setXAstrom(s1.getXAstrom() + random.uniform(-.1, .1))
catSet.append(s1)
#Make a SourceMatch object
maxDist = 1 / 3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "WcsVerificationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(4000, 4000)))
def testSingleExposure(self):
policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline",
"DiffImStageDictionary.paf", "policy")
policy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True)
self.subBackground(policy.get("diffImPolicy").get("backgroundPolicy"))
stage = ipPipe.DiffImStage(policy)
tester = SimpleStageTester(stage)
#print policy
clipboard = pexClipboard.Clipboard()
clipboard.put(policy.get("inputKeys.templateExposureKey"), self.templateExposure)
clipboard.put(policy.get("inputKeys.scienceExposureKey"), self.scienceExposure)
outWorker = tester.runWorker(clipboard)
outPolicy = policy.get("outputKeys")
#print outPolicy
self.assertTrue(outWorker.contains(outPolicy.get("differenceExposureKey")))
self.assertTrue(outWorker.contains(outPolicy.get("psfMatchingKernelKey")))
self.assertTrue(outWorker.contains(outPolicy.get("backgroundFunctionKey")))
# also check types
diffExposure = outWorker.get(outPolicy.get("differenceExposureKey"))
matchingKernel = outWorker.get(outPolicy.get("psfMatchingKernelKey"))
background = outWorker.get(outPolicy.get("backgroundFunctionKey"))
self.assertTrue(isinstance(diffExposure, afwImage.ExposureF))
self.assertTrue(isinstance(matchingKernel, afwMath.LinearCombinationKernel))
self.assertTrue(isinstance(background, afwMath.Function2D))
if display:
ds9.mtv(outWorker.get(outPolicy.get("differenceExposureKey")), frame=5)
def setUp(self):
#Load sample input from disk
srcSet = readSourceSet(
os.path.join(eups.productDir("meas_pipeline"), "tests",
"v695833-e0-c000.xy.txt"))
exp = afwImg.ExposureF(2048, 4612)
#Put them on the clipboard
fileName = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "WcsDeterminationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(fileName)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("inputExposureKey"), exp)
self.clipboard.put(self.policy.get("inputSourceSetKey"), srcSet)
# Set up local astrometry_net_data
adnDB = 'testTagAlong'
datapath = os.path.join(eups.productDir("meas_pipeline"), 'tests',
'astrometry_net_data', adnDB)
# scons doesn't set $HOME, so make sure it's set
os.environ['HOME'] = os.environ.get("HOME", os.path.expanduser("~"))
eupsObj = eups.Eups(root=datapath)
ok, version, reason = eupsObj.setup('astrometry_net_data')
if not ok:
raise ValueError(
"Need %s version of astrometry_net_data (from path: %s): %s" %
(adnDN, datapath, reason))
def testSingleExposure(self):
file = pexPolicy.DefaultPolicyFile("meas_pipeline",
"backgroundEstimation_policy.paf", "tests")
policy = pexPolicy.Policy.createPolicy(file)
stage = measPipe.BackgroundEstimationStage(policy)
tester = SimpleStageTester(stage)
clipboard = pexClipboard.Clipboard()
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
if display:
ds9.mtv(self.exposure, frame=0, title="Input")
#
# Do the work
#
outWorker = tester.runWorker(clipboard)
outPolicy = policy.get("outputKeys")
assert(outWorker.contains(outPolicy.get("backgroundSubtractedExposure")))
assert(outWorker.contains(outPolicy.get("background")))
if display:
ds9.mtv(outWorker.get(outPolicy.get("backgroundSubtractedExposure")),
frame=1, title="Subtracted")
def setUp(self):
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different fluxes.
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
#Also need to specify that each source is a star
catSet = []
flags = malgUtil.getDetectionFlags()
goodFlag = flags['BINNED1'] | flags['STAR']
for s in srcSet:
s1 = afwDet.Source(s)
s1.setPsfFlux(s1.getPsfFlux() * .281)
catSet.append(s1)
s.setFlagForDetection(goodFlag)
#Make a SourceMatch object
maxDist = 1 / 3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile("meas_pipeline",
"PhotoCalStageDictionary.paf",
"policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(10, 10)))
def testSingleExposure(self):
policyFile = pexPolicy.DefaultPolicyFile(
"ip_pipeline", "CrRejectStageDictionary.paf", "policy")
defPolicy = pexPolicy.Policy.createPolicy(
policyFile, policyFile.getRepositoryPath(), True)
policy = pexPolicy.Policy(
os.path.join(eups.productDir("ip_pipeline"), "tests",
"crReject_policy.paf"))
policy.mergeDefaults(defPolicy)
stage = ipPipe.CrRejectStage(policy)
tester = SimpleStageTester(stage)
clipboard = pexClipboard.Clipboard()
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
if display:
ds9.mtv(self.exposure, frame=0, title="Input")
#
# Do the work
#
outWorker = tester.runWorker(clipboard)
outPolicy = policy.get("outputKeys")
self.assertTrue(outWorker.contains(outPolicy.get("exposure")))
self.assertEqual(outWorker.get("nCR"), 25)
if display:
ds9.mtv(outWorker.get(outPolicy.get("exposure")),
frame=1,
title="CR removed")
def clipboardIoTest(self):
file = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "tests/sourceDetection0_policy.paf")
detectPolicy = pexPolicy.Policy.createPolicy(file)
file = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "tests/sourceMeasurement0_policy.paf")
measurePolicy = pexPolicy.Policy.createPolicy(file)
file = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "tests/psfDetermination0_policy.paf")
psfPolicy = pexPolicy.Policy.createPolicy(file)
tester = SimpleStageTester()
tester.addStage(measPipe.SourceDetectionStage(detectPolicy))
tester.addStage(measPipe.SourceMeasurementStage(measurePolicy))
tester.addStage(measPipe.PsfDeterminationStage(psfPolicy))
clipboard = pexClipboard.Clipboard()
filename = os.path.join(eups.productDir("afwdata"), "CFHT", "D4",
"cal-53535-i-797722_1")
# test only a portion of the exposure to speed up testing
bbox = afwGeom.Box2I(afwGeom.Point2I(32, 32),
afwGeom.Extent2I(512, 512))
testExp = afwImage.ExposureF(filename, 0, bbox, afwImage.LOCAL)
# test full image
# testExp = afImage.ExposureF(filename)
clipboard = pexClipboard.Clipboard()
clipboard.put(detectPolicy.get("inputKeys.exposure"), testExp)
outWorker = tester.runWorker(clipboard)
assert (outWorker.contains(psfPolicy.get("data.outputPsfKey")))
assert (outWorker.contains(psfPolicy.get("data.outputCellSetKey")))
del outWorker
del testExp
del clipboard
del tester
print >> sys.stderr, "at end of test"
def setUp(self):
afwDataDir = eups.productDir("afwdata")
self.root = afwDataDir+"/ImSim"
if not os.path.exists("registry.sqlite3"):
os.symlink(os.path.join(self.root, "registry.sqlite3"),
"./registry.sqlite3")
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put('jobIdentity', {
'visit': 85751839, 'snap': 0,
'raft': "2,3", 'sensor': "1,1", 'channel': "0,0", 'filter': "r"
})
self.clipboard.put('inputDatasets', [
Dataset("raw", visit=85751839, snap=0,
raft="2,3", sensor="1,1", channel="0,0", filter="r")
])
def testSingleInputExposure(self):
tester = SimpleStageTester()
clipboard = pexClipboard.Clipboard()
#
policy = pexPolicy.Policy.createPolicy(pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceDetection0_policy.paf",
"tests"))
tester.addStage(measPipe.SourceDetectionStage(policy))
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
#
policy = pexPolicy.Policy.createPolicy(pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceMeasurement0_policy.paf",
"tests"))
tester.addStage(measPipe.SourceMeasurementStage(policy))
#
policy = pexPolicy.Policy.createPolicy(pexPolicy.DefaultPolicyFile("meas_pipeline",
"psfDetermination_stagePolicy.paf",
"tests"))
tester.addStage(measPipe.PsfDeterminationStage(policy))
if display:
frame = 0
ds9.mtv(self.exposure, frame=frame, title="Input"); frame += 1
#
# Do the work
#
outClipboard = tester.runWorker(clipboard)
#
# See if we got it right
#
if display:
ds9.mtv(self.exposure, frame=frame, title="Detected"); frame += 1
psf = outClipboard.get(policy.get("outputKeys.psf"))
psfCellSet = outClipboard.get(policy.get("outputKeys.cellSet"))
if display:
maUtils.showPsf(psf, frame=frame); frame += 1
maUtils.showPsfMosaic(self.exposure, psf, frame=frame); frame += 1
mos = maUtils.showPsfCandidates(self.exposure, psfCellSet, frame=frame); frame += 1
del clipboard
del outClipboard
def testValidClipboard(self):
dataPolicy1 = pexPolicy.Policy()
dataPolicy1.add("inputKey", "sourceSet0")
dataPolicy1.add("outputKey", "diaSourceSet0")
dataPolicy2 = pexPolicy.Policy()
dataPolicy2.add("inputKey", "sourceSet1")
dataPolicy2.add("outputKey", "diaSourceSet1")
stagePolicy = pexPolicy.Policy()
stagePolicy.add("data", dataPolicy1)
stagePolicy.add("data", dataPolicy2)
stagePolicy.add("ccdWcsKey", "ccdWcs")
stagePolicy.add("ampBBoxKey", "ampBBox")
sourceSet = afwDet.SourceSet()
for i in xrange(5):
sourceSet.append(afwDet.Source())
point = afwGeom.Point2D(0.0, 0.0)
wcs = afwImage.makeWcs(afwCoord.Coord(point, afwGeom.degrees), point,
1., 0., 0., 1.)
ampBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1))
tester = SimpleStageTester(
measPipe.SourceToDiaSourceStage(stagePolicy))
clipboard = pexClipboard.Clipboard()
clipboard.put(dataPolicy1.get("inputKey"), sourceSet)
clipboard.put(dataPolicy2.get("inputKey"), sourceSet)
clipboard.put(stagePolicy.get("ccdWcsKey"), wcs)
clipboard.put(stagePolicy.get("ampBBoxKey"), ampBBox)
outWorker = tester.runWorker(clipboard)
for policy in stagePolicy.getPolicyArray("data"):
assert (outWorker.contains(policy.get("inputKey")))
assert (outWorker.contains(policy.get("outputKey")))
assert (outWorker.contains("persistable_" +
policy.get("outputKey")))
diaSourceSet = outWorker.get(policy.get("outputKey"))
assert (diaSourceSet.size() == sourceSet.size())
def testSingleInputExposure(self):
polFile = pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceDetection0_policy.paf",
"tests")
detectPolicy = pexPolicy.Policy.createPolicy(polFile)
polFile = pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceMeasurement0_policy.paf",
"tests")
measurePolicy = pexPolicy.Policy.createPolicy(polFile)
tester = SimpleStageTester()
tester.addStage(measPipe.SourceDetectionStage(detectPolicy))
tester.addStage(measPipe.SourceMeasurementStage(measurePolicy))
clipboard = pexClipboard.Clipboard()
clipboard.put(detectPolicy.get("inputKeys.exposure"), self.exposure)
if display:
ds9.mtv(self.exposure, frame=0, title="Input")
#
# Do the work
#
outClipboard = tester.runWorker(clipboard)
#
# See if we got it right
#
if display:
ds9.mtv(outClipboard.get(
detectPolicy.get("outputKeys.backgroundSubtractedExposure")),
frame=1,
title="Detected")
sources = measurePolicy.get("outputKeys.sources")
assert (outClipboard.contains(sources))
if False:
assert (outClipboard.contains("persistable_" + sources))
del clipboard
del outClipboard
def testSingleExposure(self):
file = pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceDetection0_policy.paf",
"tests")
policy = pexPolicy.Policy.createPolicy(file)
stage = measPipe.SourceDetectionStage(policy)
tester = SimpleStageTester(stage)
clipboard = pexClipboard.Clipboard()
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
outWorker = tester.runWorker(clipboard)
outPolicy = policy.getPolicy("outputKeys")
detectionKey = outPolicy.get("positiveDetection")
assert (outWorker.contains(detectionKey))
detectionSet = outWorker.get(detectionKey)
fpList = detectionSet.getFootprints()
assert (fpList.size() > 0)
exposureKey = outPolicy.getString("backgroundSubtractedExposure")
assert (outWorker.contains(exposureKey))
assert (outWorker.contains(outPolicy.getString("background")))
assert (outWorker.contains(outPolicy.getString("psf")))
def foo():
if len(sys.argv) == 3:
ix = int(sys.argv[1])
iy = int(sys.argv[2])
else:
ix = 0
iy = 0
afwDataDir = eups.productDir("afwdata")
obsDir = eups.productDir('obs_lsstSim')
isrDir = eups.productDir('ip_isr')
pipelineDir = eups.productDir('ip_pipeline')
lutpolicypath = os.path.join(isrDir, "pipeline")
pipepolicypath = os.path.join(pipelineDir, "policy")
dc3MetadataFile = 'dc3MetadataPolicy.paf'
simDatatypeFile = 'simDataTypePolicy.paf'
suffix = "Keyword"
"""Pipeline test case."""
clipboard = pexClipboard.Clipboard()
clipboard.put(
'jobIdentity', {
'visit': 2,
'snap': 0,
'raft': "R:2,3",
'sensor': "S:1,1",
'channel': "%i%i" % (ix, iy)
})
clipboard.put('inputDatasets', [
Dataset("raw",
visit=2,
snap=0,
raft="R:2,3",
sensor="S:1,1",
channel="%i%i" % (ix, iy))
])
clipboard.put("fwhm", 5.)
p0 = pexPolicy.Policy.createPolicy("MakeCalibInputStage.paf")
s0 = lsst.pex.harness.IOStage.InputStage(p0)
t0 = SimpleStageTester(s0)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrSaturationStageDictionary.paf",
"policy")
p2 = pexPolicy.Policy.createPolicy(file)
s2 = ipPipe.IsrSaturationStage(p2)
t2 = SimpleStageTester(s2)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrOverscanStageDictionary.paf",
"policy")
p3 = pexPolicy.Policy.createPolicy(file)
clipboard.put(p3.get("inputKeys.overscanfittype"), "MEDIAN")
s3 = ipPipe.IsrOverscanStage(p3)
t3 = SimpleStageTester(s3)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrBiasStageDictionary.paf", "policy")
p4 = pexPolicy.Policy.createPolicy(file)
s4 = ipPipe.IsrBiasStage(p4)
t4 = SimpleStageTester(s4)
file = pexPolicy.DefaultPolicyFile("ip_pipeline",
"IsrDarkStageDictionary.paf", "policy")
p5 = pexPolicy.Policy.createPolicy(file)
s5 = ipPipe.IsrDarkStage(p5)
t5 = SimpleStageTester(s5)
o0 = t0.runWorker(clipboard)
o2 = t2.runWorker(o0)
if display:
ds9.mtv(o2.get(p2.get("outputKeys.saturationCorrectedExposure")),
frame=0,
title="Sat")
o2.put(p3.get("inputKeys.exposure"),
o2.get(p2.get("outputKeys.saturationCorrectedExposure")))
o3 = t3.runWorker(o2)
if display:
ds9.mtv(o3.get(p3.get("outputKeys.overscanCorrectedExposure")),
frame=1,
title="Overscan")
o3.put(p4.get("inputKeys.exposure"),
o3.get(p3.get("outputKeys.overscanCorrectedExposure")))
o4 = t4.runWorker(o3)
if display:
ds9.mtv(o4.get(p4.get("outputKeys.biasSubtractedExposure")),
frame=3,
title="Bias")
o4.put(p5.get("inputKeys.exposure"),
o4.get(p4.get("outputKeys.biasSubtractedExposure")))
o5 = t5.runWorker(o4)
if display:
ds9.mtv(o5.get(p5.get("outputKeys.darkSubtractedExposure")),
frame=4,
title="Dark")
exposure = o5.get(p5.get("outputKeys.darkSubtractedExposure"))
exposure.writeFits("flat%i%i.fits" % (ix, iy))
if display:
ds9.mtv(exposure, frame=5, title="Output")
def testSingleInputExposure(self):
tester = SimpleStageTester()
clipboard = pexClipboard.Clipboard()
#
policy = pexPolicy.Policy.createPolicy(
pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceDetection0_policy.paf",
"tests"))
tester.addStage(measPipe.SourceDetectionStage(policy))
clipboard.put(policy.get("inputKeys.exposure"), self.exposure)
#
policy = pexPolicy.Policy.createPolicy(
pexPolicy.DefaultPolicyFile("meas_pipeline",
"sourceMeasurement0_policy.paf",
"tests"))
tester.addStage(measPipe.SourceMeasurementStage(policy))
#
policy = pexPolicy.Policy.createPolicy(
pexPolicy.DefaultPolicyFile("meas_pipeline",
"psfDetermination_stagePolicy.paf",
"tests"))
tester.addStage(measPipe.PsfDeterminationStage(policy))
# create the aperture correction object, for later use
acPolicy = pexPolicy.Policy.createPolicy(
pexPolicy.DefaultPolicyFile("meas_pipeline",
"apCorr_stagePolicy.paf", "tests"))
tester.addStage(measPipe.ApertureCorrectionStage(acPolicy))
# apply the aperture correction in situ
acAppPolicy = pexPolicy.Policy.createPolicy(
pexPolicy.DefaultPolicyFile("meas_pipeline",
"apCorrApply_stagePolicy.paf",
"tests"))
tester.addStage(measPipe.ApertureCorrectionApplyStage(acAppPolicy))
if display:
frame = 0
ds9.mtv(self.exposure, frame=frame, title="Input")
frame += 1
#
# Do the work
#
outClipboard = tester.runWorker(clipboard)
#
# See if we got it right
#
if display:
ds9.mtv(self.exposure, frame=frame, title="Detected")
frame += 1
# check the aperture correction in the middle of the exposure
apCorr = outClipboard.get(acPolicy.get("outputKeys.apCorr"))
exposure = outClipboard.get(acPolicy.get("inputKeys.exposure"))
x, y = exposure.getWidth() / 2, exposure.getHeight() / 2
print "Mid-frame aperture correction: %.3f +/- %.3f" % apCorr.computeAt(
x, y)
# check what was applied for a few objects
sourceSet = outClipboard.get(acAppPolicy.get("inputKeys.sourceSet"))
nSource = len(sourceSet)
step = nSource / 10
for i in range(0, nSource, step):
f = sourceSet[i].getPsfFlux()
x, y = sourceSet[i].getXAstrom(), sourceSet[i].getYAstrom()
ac, acErr = apCorr.computeAt(x, y)
print "x: %7.2f y: %7.2f f: %8.2f apCorr: %5.4f+/-%5.4f" % (
x, y, f, ac, acErr)
if display:
maUtils.showPsf(psf, frame=frame)
frame += 1
maUtils.showPsfMosaic(self.exposure, psf, frame=frame)
frame += 1
mos = maUtils.showPsfCandidates(self.exposure,
psfCellSet,
frame=frame)
frame += 1
del clipboard
del outClipboard