def run(display=False):
#
# Create the task
#
config = CalibrateTask.ConfigClass()
config.initialPsf.pixelScale = 0.185 # arcsec per pixel
config.initialPsf.fwhm = 1.0
config.astrometry.retarget(MyAstrometryTask)
calibrateTask = CalibrateTask(config=config)
#
# Process the data
#
exposure = loadData(config.initialPsf.pixelScale)
result = calibrateTask.run(exposure)
exposure0, exposure = exposure, result.exposure
sources = result.sources
if display: # display on ds9 (see also --debug argparse option)
frame = 1
ds9.mtv(exposure, frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame)
def testIsSinglePrimaryFlag(self):
"""Tests detect_isPrimary column gets added when run, and that sources
labelled as detect_isPrimary are not sky sources and have no children.
"""
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(self.charImResults.exposure)
outputCat = calibResults.outputCat
self.assertTrue("detect_isPrimary" in outputCat.schema.getNames())
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])),
0)
# make sure all parent sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["deblend_nChild"] > 0)), 0)
with self.assertRaises(KeyError):
outputCat.getSchema().find("detect_isDelendedModelPrimary")
def testIsPrimaryFlag(self):
"""Tests detect_isPrimary column gets added when run, and that sources
labelled as detect_isPrimary are not sky sources and have no children.
"""
charImConfig = CharacterizeImageConfig()
charImTask = CharacterizeImageTask(config=charImConfig)
charImResults = charImTask.run(self.exposure)
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(charImResults.exposure)
outputCat = calibResults.outputCat
self.assertTrue("detect_isPrimary" in outputCat.schema.getNames())
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])),
0)
# make sure all parent sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["deblend_nChild"] > 0)), 0)
def _checkSkySourceColumnExistence(self, doSkySources):
"""Implements sky_source column checking.
Parameters
----------
doSkySource : `bool`
Value of the config flag determining whether to insert sky sources.
"""
charImConfig = CharacterizeImageConfig()
charImConfig.measurePsf.psfDeterminer = 'piff'
charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0
charImTask = CharacterizeImageTask(config=charImConfig)
charImResults = charImTask.run(self.exposure)
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibConfig.doSkySources = doSkySources
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(charImResults.exposure)
if doSkySources:
self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames())
else:
self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2")
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask), name="charImage2")
calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
isrResult1 = IsrTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get(
"postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
icResult1 = CharacterizeImageTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure,
exposureIdInfo=exposureIdInfo)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get(
"icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId,
immediate=True),
icResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground",
dataId,
immediate=True),
icResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background)
calResult1 = CalibrateTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat,
exposureIdInfo=exposureIdInfo,
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get(
"calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent"))
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground",
dataId,
immediate=True),
calResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, display=False, verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance, verbose)
#
# process the data
#
calibSources = None # sources used to calibrate the frame (photom, astrom, psf)
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, sources = result.exposure, result.sources
if returnCalibSources:
calibSources = sources
else:
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
if config.edgeRolloff.applyModel:
if verbose:
print "Adding edge rolloff distortion to the exposure WCS"
addEdgeRolloffDistortion(exposure, config.edgeRolloff)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
if display: # display on ds9 (see also --debug argparse option)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = None
frame = 1
ds9.mtv(exposure, title=os.path.split(inputFile)[1], frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame)
ds9.dot(s.getShape(), *xy, ctype=ds9.RED, frame=frame)
if radii:
for radius in radii:
ds9.dot('o', *xy, size=radius, ctype=ds9.YELLOW, frame=frame)
return exposureDict, calibSourcesDict, sourcesDict
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(
config=getObsTestConfig(IsrTask),
name="isr2"
)
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask),
name="charImage2"
)
calibrateTask = CalibrateTask(
config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema
)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
isrResult1 = IsrTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
icResult1 = CharacterizeImageTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True),
icResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True),
icResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background
)
calResult1 = CalibrateTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent")
)
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True),
calResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background
)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, displayResults=[], verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
sourceMeasurementTask.config.doApplyApCorr = 'yes'
#
# Add fields needed to identify stars while calibrating
#
keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected",
"Source was detected by calibrate")), None)]
for key in calibrateTask.getCalibKeys():
keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key))
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, calibSources = result.exposure, result.sources
else:
calibSources = None
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagateCalibFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+', *xy,
ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
algMetaData = dafBase.PropertyList()
measureTask = SourceMeasurementTask(schema, algMetaData, config=meas_config)
exposure = afwImage.ExposureF(image_file)
#
# Add variance, inferred from the data, to the exposure
#
mi = exposure.getMaskedImage()
varclip = afwMath.makeStatistics(mi.getImage(),
afwMath.VARIANCECLIP).getValue()
mi.getVariance().getArray()[:] = np.ones(mi.getArrays()[0].shape)*varclip
#
# Run calibration task
#
result = calibrateTask.run(exposure)
exposure, calibSources = result.exposure, result.sources
#
# Create XYTransform for the edge rolloff effect
#
warper = EdgeRolloffWarper(2, 30, 400, 2, 30, 400)
transform = warper.transform.clone()
distortedTanWcs = afwImage.DistortedTanWcs(afwImage.TanWcs.cast(exposure.getWcs()), transform)
#output_table = afwTable.SourceTable.make(schema)
#sources = detectionTask.run(output_table, exposure, sigma=5).sources
#
## Disable psfFlux since it is crashing.
#measureTask.config.slots.psfFlux = None
#measureTask.run(exposure, calibSources)
