def processCcd(REPO,visit,ccdnum,**kw):
from lsst.pipe.tasks.processCcd import ProcessCcdTask
command = "{REPO} --id visit={EXPNUM} ccdnum={CCDNUM} --output {output} --configfile {configfile}"
args = command.format(REPO=REPO,EXPNUM=visit,CCDNUM=ccdnum,**kw).split()
print "Running:\n%s\n" % " ".join(args)
ProcessCcdTask.parseAndRun(args=args)
return
def setUpClass(cls):
"""Runs ProcessCcdTask so the test* methods can inspect the results."""
try:
cls.datadir = getPackageDir("testdata_decam")
except pexExcept.NotFoundError:
message = "testdata_decam not setup. Skipping."
warnings.warn(message)
raise unittest.SkipTest(message)
cls.outPath = tempfile.mkdtemp() if OutputName is None else OutputName
cls.dataId = {'visit': 229388, 'ccdnum': 1}
argsList = [
os.path.join(cls.datadir, "rawData"), "--output", cls.outPath,
"--id"
]
argsList += ["%s=%s" % (key, val) for key, val in cls.dataId.items()]
argsList += ["--calib", os.path.join(cls.datadir, "rawData/cpCalib")]
argsList += [
"--config", "calibrate.doPhotoCal=False",
"calibrate.doAstrometry=False"
]
argsList.append('--doraise')
disableImplicitThreading() # avoid contention with other processes
fullResult = ProcessCcdTask.parseAndRun(args=argsList,
doReturnResults=True)
cls.butler = fullResult.parsedCmd.butler
cls.config = fullResult.parsedCmd.config
def py_pccd_process_by_id(repo: RepoInfo, id: str, stdout=parsl.AUTO_LOGNAME):
argslist = repo.cli_as_list()
print("BENC ARGS LIST 1: {}".format(argslist))
if id:
argslist += ["--id", id]
else:
argslist += ["--id"]
print("BENC ARGS LIST 2: {}".format(argslist))
from lsst.pipe.tasks.processCcd import ProcessCcdTask
try:
ProcessCcdTask.parseAndRun(args=argslist)
except SystemExit as e:
raise ValueError("SystemExit {} from ProcessCcdTask".format(e.code))
def testInterface(self):
obsTestDir = lsst.utils.getPackageDir('obs_test')
inputDir = os.path.join(obsTestDir, "data", "input")
# Configure a ProcessCcd task such that it will return a minimal
# number of measurements plus our test plugin.
cfg = ProcessCcdConfig()
cfg.calibrate.measurement.plugins.names = ["base_SdssCentroid", "base_SkyCoord", PLUGIN_NAME]
cfg.calibrate.measurement.slots.shape = None
cfg.calibrate.measurement.slots.psfFlux = None
cfg.calibrate.measurement.slots.apFlux = None
cfg.calibrate.measurement.slots.gaussianFlux = None
cfg.calibrate.measurement.slots.modelFlux = None
cfg.calibrate.measurement.slots.calibFlux = None
# no reference catalog, so...
cfg.calibrate.doAstrometry = False
cfg.calibrate.doPhotoCal = False
# disable aperture correction because we aren't measuring aperture flux
cfg.calibrate.doApCorr = False
# Extendedness requires modelFlux, disabled above.
cfg.calibrate.catalogCalculation.plugins.names.discard("base_ClassificationExtendedness")
# Process the test data with ProcessCcd then perform a transform.
with tempDirectory() as tempDir:
measResult = ProcessCcdTask.parseAndRun(args=[inputDir, "--output", tempDir, "--id", "visit=1"],
config=cfg, doReturnResults=True)
trArgs = [tempDir, "--output", tempDir, "--id", "visit=1",
"-c", "inputConfigType=processCcd_config"]
trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True)
# It should be possible to reprocess the data through a new transform task with exactly
# the same configuration without throwing. This check is useful since we are
# constructing the task on the fly, which could conceivably cause problems with
# configuration/metadata persistence.
trResult = SrcTransformTask.parseAndRun(args=trArgs, doReturnResults=True)
measSrcs = measResult.resultList[0].result.calibRes.sourceCat
trSrcs = trResult.resultList[0].result
# The length of the measured and transformed catalogs should be the same.
self.assertEqual(len(measSrcs), len(trSrcs))
# Each source should have been measured & transformed appropriately.
for measSrc, trSrc in zip(measSrcs, trSrcs):
# The TrivialMeasurement should be transformed as defined above.
self.assertEqual(trSrc[PLUGIN_NAME], measSrc[PLUGIN_NAME])
self.assertEqual(trSrc[PLUGIN_NAME + "_transform"], -1.0 * measSrc[PLUGIN_NAME])
# The SdssCentroid should be transformed to celestial coordinates.
# Checking that the full transformation has been done correctly is
# out of scope for this test case; we just ensure that there's
# plausible position in the transformed record.
trCoord = afwTable.CoordKey(trSrcs.schema["base_SdssCentroid"]).get(trSrc)
self.assertAlmostEqual(measSrc.getCoord().getLongitude(), trCoord.getLongitude())
self.assertAlmostEqual(measSrc.getCoord().getLatitude(), trCoord.getLatitude())
def testFakeProcessing(self):
# Set the random seed for predictability
np.random.seed(500)
# Set ouput path and create a dataId
outPath = tempfile.mkdtemp() if OutputName is None \
else "{}-ProcessCcd".format(OutputName)
dataId = dict(visit=1)
dataIdStrList = ["{}={}".format(*item) for item in dataId.items()]
mask = None
maskPlaneName = "FAKE"
try:
# Set the configurations for running the fake object piepline
processCcdConfig = getObsTestConfig(ProcessCcdTask)
processCcdConfig.calibrate.doInsertFakes = True
processCcdConfig.calibrate.insertFakes.retarget(
FakeSourcesTestTask)
# Run ProcessCcd
pCcdResult = ProcessCcdTask.parseAndRun(args=[
InputDir, "--output", outPath, "--clobber-config", "--doraise",
"--id"
] + dataIdStrList,
doReturnResults=True,
config=processCcdConfig)
# Check the Catalog contains properly measured fake sources
sourceCat = pCcdResult.resultList[0].result.calibRes.sourceCat
self.checkSourceCatalog(sourceCat)
exposure = pCcdResult.resultList[0].result.calibRes.exposure
mask = exposure.getMaskedImage().getMask()
maskBit = 2**mask.getMaskPlaneDict()[maskPlaneName]
fakeMask = np.bitwise_and(mask.getArray(), maskBit)
self.assertGreater(fakeMask.sum(), 0)
# Clean up temporary directories if needed
finally:
if mask:
# Remove FAKE so as not to contaminate later tests
lsst.afw.image.Mask[lsst.afw.image.MaskPixel].removeMaskPlane(
maskPlaneName)
if OutputName is None:
shutil.rmtree(outPath)
else:
message = "testFakeProcessing.py's output data saved to {}"
print(message.format(OutputName))
def testFakeProcessing(self):
# Set the random seed for predictability
np.random.seed(500)
# Set ouput path and create a dataId
outPath = tempfile.mkdtemp() if OutputName is None \
else "{}-ProcessCcd".format(OutputName)
dataId = dict(visit=1)
dataIdStrList = ["{}={}".format(*item) for item in dataId.items()]
mask = None
maskPlaneName = "FAKE"
try:
# Set the configurations for running the fake object piepline
processCcdConfig = getObsTestConfig(ProcessCcdTask)
processCcdConfig.calibrate.doInsertFakes = True
processCcdConfig.calibrate.insertFakes.retarget(FakeSourcesTestTask)
# Run ProcessCcd
pCcdResult = ProcessCcdTask.parseAndRun(
args=[InputDir, "--output", outPath,
"--clobber-config", "--doraise", "--id"] +
dataIdStrList,
doReturnResults=True, config=processCcdConfig)
# Check the Catalog contains properly measured fake sources
sourceCat = pCcdResult.resultList[0].result.calibRes.sourceCat
self.checkSourceCatalog(sourceCat)
exposure = pCcdResult.resultList[0].result.calibRes.exposure
mask = exposure.getMaskedImage().getMask()
maskBit = 2**mask.getMaskPlaneDict()[maskPlaneName]
fakeMask = np.bitwise_and(mask.getArray(), maskBit)
self.assertGreater(fakeMask.sum(), 0)
# Clean up temporary directories if needed
finally:
if mask:
# Remove FAKE so as not to contaminate later tests
lsst.afw.image.Mask[lsst.afw.image.MaskPixel].removeMaskPlane(maskPlaneName)
if OutputName is None:
shutil.rmtree(outPath)
else:
message = "testFakeProcessing.py's output data saved to {}"
print(message.format(OutputName))
def setUp(self):
try:
self.datadir = getPackageDir("testdata_decam")
except pexExcept.NotFoundError:
message = "testdata_decam not setup. Skipping."
warnings.warn(message)
raise unittest.SkipTest(message)
self.outPath = tempfile.mkdtemp() if OutputName is None else OutputName
self.dataId = {'visit': 229388, 'ccdnum': 1}
argsList = [os.path.join(self.datadir, "rawData"), "--output", self.outPath, "--id"]
argsList += ["%s=%s" % (key, val) for key, val in self.dataId.iteritems()]
argsList += ["--config", "calibrate.doPhotoCal=False", "calibrate.doAstrometry=False",
# Temporary until DM-4232 is fixed.
"isr.assembleCcd.setGain=False"]
fullResult = ProcessCcdTask.parseAndRun(args=argsList, doReturnResults=True)
self.butler = fullResult.parsedCmd.butler
self.config = fullResult.parsedCmd.config
def testProcessCcd(self):
"""test ProcessCcdTask via parseAndRun (simulating the command line)
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-ProcessCcd".format(OutputName)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
fullResult = ProcessCcdTask.parseAndRun(args=[
InputDir, "--output", outPath, "--clobber-config", "--doraise",
"-c", "charImage.doWriteExposure=True", "--id"
] + dataIdStrList,
doReturnResults=True)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
icExpBackground = butler.get("icExpBackground",
dataId,
immediate=True)
bg0Arr = icExpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=np.float64)
bgStdDev = bg0Arr.std(dtype=np.float64)
icSrc = butler.get("icSrc", dataId)
src = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate(
(butler.get("calexp", dataId), result.exposure)):
self.assertEqual(
exposure.getBBox(),
geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = np.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=np.float64)
imStdDev = imageArr.std(dtype=np.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=np.float64)
varStdDev = varArr.std(dtype=np.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" %
(bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(src) :", len(src))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" %
(varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" %
(psfIxx, psfIyy, psfIxy))
self.assertEqual(len(icSrc), 28)
self.assertEqual(
len(src),
185 + 100) # 185 real sources plus 100 sky sources
expectedPlaces = 7 # Tolerance for numerical comparisons
for name, var, val in [
("bgMean", bgMean, 191.48635852060525),
("bgStdDev", bgStdDev, 0.2399466881603354),
("numGoodPix", numGoodPix, 1966820),
("imMean", imMean, 1.1237668985230562),
("imStdDev", imStdDev, 85.81296241298496),
("varMean", varMean, 131.24003624152013),
("varStdDev", varStdDev, 55.98012493452948),
("psfIxx", psfIxx, 2.769679536557131),
("psfIyy", psfIyy, 2.2013649766299324),
("psfIxy", psfIxy, 0.14797939531970852)
]:
self.assertAlmostEqual(var,
val,
places=expectedPlaces,
msg=name)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def testProcessCcd(self):
"""test ProcessCcdTask
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp() if OutputName is None else OutputName
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.iteritems()
]
fullResult = ProcessCcdTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config", "--id"
] + dataIdStrList,
doReturnResults=True,
)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
icExpBackground = butler.get("icExpBackground",
dataId,
immediate=True)
bg0Arr = icExpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=numpy.float64)
bgStdDev = bg0Arr.std(dtype=numpy.float64)
icSrc = butler.get("icSrc", dataId)
src = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate((
butler.get("calexp", dataId),
result.exposure,
)):
self.assertEqual(
exposure.getBBox(),
afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = numpy.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=numpy.float64)
imStdDev = imageArr.std(dtype=numpy.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=numpy.float64)
varStdDev = varArr.std(dtype=numpy.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" %
(bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(src) :", len(src))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" %
(varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" %
(psfIxx, psfIyy, psfIxy))
self.assertAlmostEqual(bgMean,
191.51202961532354,
places=7)
self.assertAlmostEqual(bgStdDev,
0.28333327656694213,
places=7)
self.assertEqual(len(icSrc), 28)
self.assertEqual(len(src), 185)
self.assertEqual(numGoodPix, 1965540)
self.assertAlmostEqual(imMean,
0.99578990765845121,
places=7)
self.assertAlmostEqual(imStdDev,
95.645409033639211,
places=7)
self.assertAlmostEqual(varMean,
131.16293718847217,
places=7)
self.assertAlmostEqual(varStdDev,
64.806576059889963,
places=7)
self.assertAlmostEqual(psfIxx,
2.8540775242108163,
places=7)
self.assertAlmostEqual(psfIyy, 2.17386182921239, places=7)
self.assertAlmostEqual(psfIxy,
0.14400008637208778,
places=7)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def testProcessCcd(self):
"""test ProcessCcdTask via parseAndRun (simulating the command line)
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-ProcessCcd".format(OutputName)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
fullResult = ProcessCcdTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise",
"-c", "charImage.doWriteExposure=True", "--id"] + dataIdStrList,
doReturnResults=True
)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
icExpBackground = butler.get("icExpBackground", dataId, immediate=True)
bg0Arr = icExpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=np.float64)
bgStdDev = bg0Arr.std(dtype=np.float64)
icSrc = butler.get("icSrc", dataId)
src = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate((butler.get("calexp", dataId), result.exposure)):
self.assertEqual(exposure.getBBox(),
afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = np.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=np.float64)
imStdDev = imageArr.std(dtype=np.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=np.float64)
varStdDev = varArr.std(dtype=np.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(src) :", len(src))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" % (varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy))
self.assertEqual(len(icSrc), 28)
self.assertEqual(len(src), 186)
expectedPlaces = 7 # Tolerance for numerical comparisons
for name, var, val in [
("bgMean", bgMean, 191.4866883539376),
("bgStdDev", bgStdDev, 0.24004985195839124),
("numGoodPix", numGoodPix, 1965472),
("imMean", imMean, 0.99616767174645282),
("imStdDev", imStdDev, 95.64615285227903),
("varMean", varMean, 131.14644188960244),
("varStdDev", varStdDev, 64.806980097108436),
("psfIxx", psfIxx, 2.8540502512317234),
("psfIyy", psfIyy, 2.17386622227748),
("psfIxy", psfIxy, 0.14397638586604858)
]:
self.assertAlmostEqual(var, val, places=expectedPlaces, msg=name)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def testProcessCcd(self):
"""test ProcessCcdTask via parseAndRun (simulating the command line)
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-ProcessCcd".format(OutputName)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
fullResult = ProcessCcdTask.parseAndRun(args=[
InputDir, "--output", outPath, "--clobber-config", "--doraise",
"-c", "charImage.doWriteExposure=True", "--id"
] + dataIdStrList,
doReturnResults=True)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
icExpBackground = butler.get("icExpBackground",
dataId,
immediate=True)
bg0Arr = icExpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=np.float64)
bgStdDev = bg0Arr.std(dtype=np.float64)
icSrc = butler.get("icSrc", dataId)
src = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate(
(butler.get("calexp", dataId), result.exposure)):
self.assertEqual(
exposure.getBBox(),
geom.Box2I(geom.Point2I(0, 0), geom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = np.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=np.float64)
imStdDev = imageArr.std(dtype=np.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=np.float64)
varStdDev = varArr.std(dtype=np.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" %
(bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(src) :", len(src))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" %
(varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" %
(psfIxx, psfIyy, psfIxy))
self.assertEqual(len(icSrc), 28)
self.assertEqual(
len(src),
184 + 100) # 184 real sources plus 100 sky sources
# NOTE: These values are purely empirical, and need to be
# updated to reflect major algorithmic changes.
# If this test fails after an algorithmic change due to
# small numeric changes here, check on slack at
# #dm-science-pipelines as to whether the changes are
# reasonable, and then replace the failing values by
# running the test to determine the updated values.
expectedPlaces = 7 # Tolerance for numerical comparisons
for name, var, val in [
("bgMean", bgMean, 191.48623786891795),
("bgStdDev", bgStdDev, 0.23994185672586282),
("numGoodPix", numGoodPix, 1966606),
("imMean", imMean, 1.1242456954648634),
("imStdDev", imStdDev, 85.8129750182329),
("varMean", varMean, 131.24003624152013),
("varStdDev", varStdDev, 55.98012493452948),
("psfIxx", psfIxx, 2.843329671276296),
("psfIyy", psfIyy, 2.2249941554078156),
("psfIxy", psfIxy, 0.16073332780683286)
]:
self.assertAlmostEqual(var,
val,
places=expectedPlaces,
msg=name)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def testProcessCcd(self):
"""test ProcessCcdTask
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp() if OutputName is None else OutputName
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.iteritems()]
fullResult = ProcessCcdTask.parseAndRun(
args = [InputDir, "--output", outPath, "--id"] + dataIdStrList,
doReturnResults = True,
)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
calexpBackground = butler.get("calexpBackground", dataId)
bg0Arr = calexpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=numpy.float64)
bgStdDev = bg0Arr.std(dtype=numpy.float64)
icSrc = butler.get("icSrc", dataId)
sources = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate((
butler.get("calexp", dataId),
result.exposure,
)):
self.assertEqual(exposure.getBBox(),
afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = numpy.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=numpy.float64)
imStdDev = imageArr.std(dtype=numpy.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=numpy.float64)
varStdDev = varArr.std(dtype=numpy.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(sources) =", len(sources))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" % (varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy))
self.assertAlmostEqual(bgMean, 327.61695151065607, places=7)
self.assertAlmostEqual(bgStdDev, 0.40081096398879795, places=7)
self.assertEqual(len(icSrc), 28)
self.assertEqual(len(sources), 169)
self.assertEqual(numGoodPix, 1953514)
self.assertAlmostEqual(imMean, 1.7045581411166495, places=7)
self.assertAlmostEqual(imStdDev, 163.66290334994014, places=7)
self.assertAlmostEqual(varMean, 212.5416436210289, places=7)
self.assertAlmostEqual(varStdDev, 121.25084470503148, places=7)
self.assertAlmostEqual(psfIxx, 2.790601924907123, places=7)
self.assertAlmostEqual(psfIyy, 2.1989110429762513, places=7)
self.assertAlmostEqual(psfIxy, 0.16599586021207952, places=7)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
#!/usr/bin/env python # # LSST Data Management System # Copyright 2008, 2009, 2010 LSST Corporation. # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # from lsst.pipe.tasks.processCcd import ProcessCcdTask ProcessCcdTask.parseAndRun()
def testProcessCcd(self):
"""test ProcessCcdTask via parseAndRun (simulating the command line)
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-ProcessCcd".format(OutputName)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
fullResult = ProcessCcdTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise",
"-c", "charImage.doWriteExposure=True", "--id"] + dataIdStrList,
doReturnResults=True
)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
icExpBackground = butler.get("icExpBackground", dataId, immediate=True)
bg0Arr = icExpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=np.float64)
bgStdDev = bg0Arr.std(dtype=np.float64)
icSrc = butler.get("icSrc", dataId)
src = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate((butler.get("calexp", dataId), result.exposure)):
self.assertEqual(exposure.getBBox(),
afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = np.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=np.float64)
imStdDev = imageArr.std(dtype=np.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=np.float64)
varStdDev = varArr.std(dtype=np.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(src) :", len(src))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" % (varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy))
self.assertEqual(len(icSrc), 28)
self.assertEqual(len(src), 185)
expectedPlaces = 7 # Tolerance for numerical comparisons
for var, val in [
(bgMean, 191.51453611409124),
(bgStdDev, 0.22438381414455047),
(numGoodPix, 1965508),
(imMean, 0.99592485493752636),
(imStdDev, 95.64609939459902),
(varMean, 131.16293718847217),
(varStdDev, 64.806576059889963),
(psfIxx, 2.8540480723051846),
(psfIyy, 2.173868563513369),
(psfIxy, 0.14397457739362085)
]:
self.assertAlmostEqual(var, val, places=expectedPlaces)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def testProcessCcd(self):
"""test ProcessCcdTask
This is intended as a sanity check of the task, not a detailed test of its sub-tasks. As such
comparisons are intentionally loose, so as to allow evolution of the sub-tasks over time
without breaking this test.
"""
outPath = tempfile.mkdtemp() if OutputName is None else OutputName
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.iteritems()]
fullResult = ProcessCcdTask.parseAndRun(
args = [InputDir, "--output", outPath, "--clobber-config", "--id"] + dataIdStrList,
doReturnResults = True,
)
butler = fullResult.parsedCmd.butler
self.assertEqual(len(fullResult.resultList), 1)
runResult = fullResult.resultList[0]
fullDataId = runResult.dataRef.dataId
self.assertEqual(len(fullDataId), 2)
self.assertEqual(fullDataId["visit"], dataId["visit"])
self.assertEqual(fullDataId["filter"], "g")
result = runResult.result
icExpBackground = butler.get("icExpBackground", dataId, immediate=True)
bg0Arr = icExpBackground.getImage().getArray()
bgMean = bg0Arr.mean(dtype=numpy.float64)
bgStdDev = bg0Arr.std(dtype=numpy.float64)
icSrc = butler.get("icSrc", dataId)
src = butler.get("src", dataId)
# the following makes pyflakes linter happy and the code more robust
oldImMean = None
oldImStdDev = None
oldVarMean = None
oldVarStdDev = None
oldPsfIxx = None
oldPsfIyy = None
oldPsfIxy = None
for i, exposure in enumerate((
butler.get("calexp", dataId),
result.exposure,
)):
self.assertEqual(exposure.getBBox(),
afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1018, 2000)))
maskedImage = exposure.getMaskedImage()
maskArr = maskedImage.getMask().getArray()
numGoodPix = numpy.sum(maskArr == 0)
imageArr = maskedImage.getImage().getArray()
imMean = imageArr.mean(dtype=numpy.float64)
imStdDev = imageArr.std(dtype=numpy.float64)
varArr = maskedImage.getVariance().getArray()
varMean = varArr.mean(dtype=numpy.float64)
varStdDev = varArr.std(dtype=numpy.float64)
psfShape = exposure.getPsf().computeShape()
psfIxx = psfShape.getIxx()
psfIyy = psfShape.getIyy()
psfIxy = psfShape.getIxy()
if i == 0:
print("\nMeasured results:")
print("background mean = %r, stdDev = %r" % (bgMean, bgStdDev))
print("len(icSrc) :", len(icSrc))
print("len(src) :", len(src))
print("numGoodPix =", numGoodPix)
print("image mean = %r, stdDev = %r" % (imMean, imStdDev))
print("variance mean = %r, stdDev = %r" % (varMean, varStdDev))
print("psf Ixx = %r, Iyy = %r, Ixy = %r" % (psfIxx, psfIyy, psfIxy))
self.assertAlmostEqual(bgMean, 191.51202961532354, places=7)
self.assertAlmostEqual(bgStdDev, 0.28333327656694213, places=7)
self.assertEqual(len(icSrc), 28)
self.assertEqual(len(src), 185)
self.assertEqual(numGoodPix, 1965540)
self.assertAlmostEqual(imMean, 0.99578990765845121, places=7)
self.assertAlmostEqual(imStdDev, 95.645409033639211, places=7)
self.assertAlmostEqual(varMean, 131.16293718847217, places=7)
self.assertAlmostEqual(varStdDev, 64.806576059889963, places=7)
self.assertAlmostEqual(psfIxx, 2.8540775242108163, places=7)
self.assertAlmostEqual(psfIyy, 2.17386182921239, places=7)
self.assertAlmostEqual(psfIxy, 0.14400008637208778, places=7)
else:
self.assertEqual(imMean, oldImMean)
self.assertEqual(imStdDev, oldImStdDev)
self.assertEqual(varMean, oldVarMean)
self.assertEqual(varStdDev, oldVarStdDev)
self.assertEqual(psfIxx, oldPsfIxx)
self.assertEqual(psfIyy, oldPsfIyy)
self.assertEqual(psfIxy, oldPsfIxy)
oldImMean = imMean
oldImStdDev = imStdDev
oldVarMean = varMean
oldVarStdDev = varStdDev
oldPsfIxx = psfIxx
oldPsfIyy = psfIyy
oldPsfIxy = psfIxy
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def __init__(self, **kwargs):
"""Construct a ProcessEimageTask
"""
ProcessCcdTask.__init__(self, **kwargs)
numpy.random.seed(self.config.rngSeed)
