def setUp(self): # Load sample input from disk expPath = os.path.join(getPackageDir("pipe_tasks"), "tests", "data", "v695833-e0-c000-a00.sci.fits") self.exposure = afwImage.ExposureF(expPath) # Characterize the image (create PSF, etc.) charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) self.charImResults = charImTask.run(self.exposure) # set log level so that warnings do not display Log.getLogger("calibrate").setLevel(Log.ERROR)
def run(self, sensor_id, infile, gains, bias_frame=None): # # Process a CCD image mosaic # if self.config.verbose: self.log.info("processing {0}".format(infile)) image = make_ccd_mosaic(infile, bias_frame=bias_frame, gains=gains) exposure = afwImage.ExposureF(image.getBBox()) exposure.setImage(image) # # Set up characterize task configuration # nsig = self.config.nsig bgbinsize = self.config.bgbinsize minpixels = self.config.minpixels charConfig = CharacterizeImageConfig() charConfig.doMeasurePsf = False charConfig.doApCorr = False charConfig.repair.doCosmicRay = False charConfig.detection.minPixels = minpixels charConfig.detection.background.binSize = bgbinsize charConfig.detection.thresholdType = "stdev" charConfig.detection.thresholdValue = nsig hsm_plugins = set(["ext_shapeHSM_HsmShapeBj", "ext_shapeHSM_HsmShapeLinear", "ext_shapeHSM_HsmShapeKsb", "ext_shapeHSM_HsmShapeRegauss", "ext_shapeHSM_HsmSourceMoments", "ext_shapeHSM_HsmPsfMoments"]) charConfig.measurement.plugins.names |= hsm_plugins charTask = CharacterizeImageTask(config=charConfig) if lsst.pipe.tasks.__version__.startswith('17.0'): result = charTask.run(exposure) else: result = charTask.characterize(exposure) src = result.sourceCat if self.config.verbose: self.log.info("Detected {0} objects".format(len(src))) # # Save catalog results to file # output_dir = self.config.output_dir if self.config.output_file is None: output_file = os.path.join(output_dir, '{0}_source_catalog.cat'.format(sensor_id)) else: output_file = os.path.join(output_dir, self.config.output_file) if self.config.verbose: self.log.info("Writing spot results file to {0}".format(output_file)) src.writeFits(output_file)
def testFlags(self): # Test that all of the flags are defined and there is no reservation by default # also test that used sources are a subset of candidate sources task = CharacterizeImageTask() results = task.run(self.exposure) used = 0 reserved = 0 for source in results.sourceCat: if source.get("calib_psf_used"): used += 1 self.assertTrue(source.get("calib_psf_candidate")) if source.get("calib_psf_reserved"): reserved += 1 self.assertGreater(used, 0) self.assertEqual(reserved, 0)
def testFlags(self): # Test that all of the flags are defined and there is no reservation by default # also test that used sources are a subset of candidate sources config = CharacterizeImageConfig() config.measurePsf.psfDeterminer = 'piff' config.measurePsf.psfDeterminer['piff'].spatialOrder = 0 task = CharacterizeImageTask(config=config) results = task.run(self.exposure) used = 0 reserved = 0 for source in results.sourceCat: if source.get("calib_psf_used"): used += 1 self.assertTrue(source.get("calib_psf_candidate")) if source.get("calib_psf_reserved"): reserved += 1 self.assertGreater(used, 0) self.assertEqual(reserved, 0)
def testIsPrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0)
def _checkSkySourceColumnExistence(self, doSkySources): """Implements sky_source column checking. Parameters ---------- doSkySource : `bool` Value of the config flag determining whether to insert sky sources. """ charImConfig = CharacterizeImageConfig() charImConfig.measurePsf.psfDeterminer = 'piff' charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0 charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibConfig.doSkySources = doSkySources calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) if doSkySources: self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames()) else: self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2") # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2") calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] isrResult1 = IsrTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get( "postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) icResult1 = CharacterizeImageTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure, exposureIdInfo=exposureIdInfo) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get( "icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background) calResult1 = CalibrateTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat, exposureIdInfo=exposureIdInfo, ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get( "calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual(calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent")) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask( config=getObsTestConfig(IsrTask), name="isr2" ) # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2" ) calibrateTask = CalibrateTask( config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema ) try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()] isrResult1 = IsrTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) icResult1 = CharacterizeImageTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background ) calResult1 = CalibrateTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") ) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background ) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
img = exposure_copy.image sim = img.Factory(img, amp.getBBox()) sim *= gain print(amp.getName(), gain, amp.getBBox()) sys.stdout.flush() if do_bf_corr: brighterFatterCorrection(exposure_copy[amp.getBBox()],bf_kernel.kernel[amp.getName()],20,10,False) else: continue # Now trim the exposure down to the region of interest trimmed_exposure = exposure_copy[spots_bbox] # Now find and characterize the spots charTask = CharacterizeImageTask(config=charConfig) tstart=time.time() charResult = charTask.run(trimmed_exposure) spotCatalog = charResult.sourceCat print("%s, Correction = %r, Characterization took "%(amp.getName(),do_bf_corr),str(time.time()-tstart)[:4]," seconds") sys.stdout.flush() # Now trim out spots not between minSize and maxSize select = ((spotCatalog['base_SdssShape_xx'] >= minSize) & (spotCatalog['base_SdssShape_xx'] <= maxSize) & (spotCatalog['base_SdssShape_yy'] >= minSize) & (spotCatalog['base_SdssShape_yy'] <= maxSize)) spotCatalog = spotCatalog.subset(select) x2 = spotCatalog['base_SdssShape_xx'] y2 = spotCatalog['base_SdssShape_yy'] flux = spotCatalog['base_SdssShape_instFlux'] numspots = len(flux) print("Detected ",len(spotCatalog)," objects, Flux = %f, X2 = %f, Y2 = %f"%(np.nanmean(flux),np.nanmean(x2),np.nanmean(y2))) sys.stdout.flush() if do_bf_corr: byamp_corrected_results.append([numspots, np.nanmean(flux), np.nanstd(flux), np.nanmean(x2), np.nanstd(x2),