def setUp(self):
# Load sample input from disk
expPath = os.path.join(getPackageDir("pipe_tasks"), "tests", "data",
"v695833-e0-c000-a00.sci.fits")
self.exposure = afwImage.ExposureF(expPath)
# Characterize the image (create PSF, etc.)
charImConfig = CharacterizeImageConfig()
charImTask = CharacterizeImageTask(config=charImConfig)
self.charImResults = charImTask.run(self.exposure)
# set log level so that warnings do not display
Log.getLogger("calibrate").setLevel(Log.ERROR)
def run(self, sensor_id, infile, gains, bias_frame=None):
#
# Process a CCD image mosaic
#
if self.config.verbose:
self.log.info("processing {0}".format(infile))
image = make_ccd_mosaic(infile, bias_frame=bias_frame, gains=gains)
exposure = afwImage.ExposureF(image.getBBox())
exposure.setImage(image)
#
# Set up characterize task configuration
#
nsig = self.config.nsig
bgbinsize = self.config.bgbinsize
minpixels = self.config.minpixels
charConfig = CharacterizeImageConfig()
charConfig.doMeasurePsf = False
charConfig.doApCorr = False
charConfig.repair.doCosmicRay = False
charConfig.detection.minPixels = minpixels
charConfig.detection.background.binSize = bgbinsize
charConfig.detection.thresholdType = "stdev"
charConfig.detection.thresholdValue = nsig
hsm_plugins = set(["ext_shapeHSM_HsmShapeBj",
"ext_shapeHSM_HsmShapeLinear",
"ext_shapeHSM_HsmShapeKsb",
"ext_shapeHSM_HsmShapeRegauss",
"ext_shapeHSM_HsmSourceMoments",
"ext_shapeHSM_HsmPsfMoments"])
charConfig.measurement.plugins.names |= hsm_plugins
charTask = CharacterizeImageTask(config=charConfig)
if lsst.pipe.tasks.__version__.startswith('17.0'):
result = charTask.run(exposure)
else:
result = charTask.characterize(exposure)
src = result.sourceCat
if self.config.verbose:
self.log.info("Detected {0} objects".format(len(src)))
#
# Save catalog results to file
#
output_dir = self.config.output_dir
if self.config.output_file is None:
output_file = os.path.join(output_dir,
'{0}_source_catalog.cat'.format(sensor_id))
else:
output_file = os.path.join(output_dir, self.config.output_file)
if self.config.verbose:
self.log.info("Writing spot results file to {0}".format(output_file))
src.writeFits(output_file)
def testFlags(self):
# Test that all of the flags are defined and there is no reservation by default
# also test that used sources are a subset of candidate sources
task = CharacterizeImageTask()
results = task.run(self.exposure)
used = 0
reserved = 0
for source in results.sourceCat:
if source.get("calib_psf_used"):
used += 1
self.assertTrue(source.get("calib_psf_candidate"))
if source.get("calib_psf_reserved"):
reserved += 1
self.assertGreater(used, 0)
self.assertEqual(reserved, 0)
def testFlags(self):
# Test that all of the flags are defined and there is no reservation by default
# also test that used sources are a subset of candidate sources
config = CharacterizeImageConfig()
config.measurePsf.psfDeterminer = 'piff'
config.measurePsf.psfDeterminer['piff'].spatialOrder = 0
task = CharacterizeImageTask(config=config)
results = task.run(self.exposure)
used = 0
reserved = 0
for source in results.sourceCat:
if source.get("calib_psf_used"):
used += 1
self.assertTrue(source.get("calib_psf_candidate"))
if source.get("calib_psf_reserved"):
reserved += 1
self.assertGreater(used, 0)
self.assertEqual(reserved, 0)
def testIsPrimaryFlag(self):
"""Tests detect_isPrimary column gets added when run, and that sources
labelled as detect_isPrimary are not sky sources and have no children.
"""
charImConfig = CharacterizeImageConfig()
charImTask = CharacterizeImageTask(config=charImConfig)
charImResults = charImTask.run(self.exposure)
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(charImResults.exposure)
outputCat = calibResults.outputCat
self.assertTrue("detect_isPrimary" in outputCat.schema.getNames())
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])),
0)
# make sure all parent sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["deblend_nChild"] > 0)), 0)
def _checkSkySourceColumnExistence(self, doSkySources):
"""Implements sky_source column checking.
Parameters
----------
doSkySource : `bool`
Value of the config flag determining whether to insert sky sources.
"""
charImConfig = CharacterizeImageConfig()
charImConfig.measurePsf.psfDeterminer = 'piff'
charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0
charImTask = CharacterizeImageTask(config=charImConfig)
charImResults = charImTask.run(self.exposure)
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibConfig.doSkySources = doSkySources
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(charImResults.exposure)
if doSkySources:
self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames())
else:
self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2")
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask), name="charImage2")
calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
isrResult1 = IsrTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get(
"postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
icResult1 = CharacterizeImageTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure,
exposureIdInfo=exposureIdInfo)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get(
"icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId,
immediate=True),
icResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground",
dataId,
immediate=True),
icResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background)
calResult1 = CalibrateTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat,
exposureIdInfo=exposureIdInfo,
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get(
"calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent"))
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground",
dataId,
immediate=True),
calResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(
config=getObsTestConfig(IsrTask),
name="isr2"
)
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask),
name="charImage2"
)
calibrateTask = CalibrateTask(
config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema
)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
isrResult1 = IsrTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
icResult1 = CharacterizeImageTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True),
icResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True),
icResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background
)
calResult1 = CalibrateTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent")
)
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True),
calResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background
)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
img = exposure_copy.image
sim = img.Factory(img, amp.getBBox())
sim *= gain
print(amp.getName(), gain, amp.getBBox())
sys.stdout.flush()
if do_bf_corr:
brighterFatterCorrection(exposure_copy[amp.getBBox()],bf_kernel.kernel[amp.getName()],20,10,False)
else:
continue
# Now trim the exposure down to the region of interest
trimmed_exposure = exposure_copy[spots_bbox]
# Now find and characterize the spots
charTask = CharacterizeImageTask(config=charConfig)
tstart=time.time()
charResult = charTask.run(trimmed_exposure)
spotCatalog = charResult.sourceCat
print("%s, Correction = %r, Characterization took "%(amp.getName(),do_bf_corr),str(time.time()-tstart)[:4]," seconds")
sys.stdout.flush()
# Now trim out spots not between minSize and maxSize
select = ((spotCatalog['base_SdssShape_xx'] >= minSize) & (spotCatalog['base_SdssShape_xx'] <= maxSize) &
(spotCatalog['base_SdssShape_yy'] >= minSize) & (spotCatalog['base_SdssShape_yy'] <= maxSize))
spotCatalog = spotCatalog.subset(select)
x2 = spotCatalog['base_SdssShape_xx']
y2 = spotCatalog['base_SdssShape_yy']
flux = spotCatalog['base_SdssShape_instFlux']
numspots = len(flux)
print("Detected ",len(spotCatalog)," objects, Flux = %f, X2 = %f, Y2 = %f"%(np.nanmean(flux),np.nanmean(x2),np.nanmean(y2)))
sys.stdout.flush()
if do_bf_corr:
byamp_corrected_results.append([numspots, np.nanmean(flux), np.nanstd(flux), np.nanmean(x2), np.nanstd(x2),
