def run(display=False): # # Create the task # config = CalibrateTask.ConfigClass() config.initialPsf.pixelScale = 0.185 # arcsec per pixel config.initialPsf.fwhm = 1.0 config.astrometry.retarget(MyAstrometryTask) calibrateTask = CalibrateTask(config=config) # # Process the data # exposure = loadData(config.initialPsf.pixelScale) result = calibrateTask.run(exposure) exposure0, exposure = exposure, result.exposure sources = result.sources if display: # display on ds9 (see also --debug argparse option) frame = 1 ds9.mtv(exposure, frame=frame) with ds9.Buffering(): for s in sources: xy = s.getCentroid() ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame)
def testIsSinglePrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(self.charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0) with self.assertRaises(KeyError): outputCat.getSchema().find("detect_isDelendedModelPrimary")
def testIsPrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0)
def _checkSkySourceColumnExistence(self, doSkySources): """Implements sky_source column checking. Parameters ---------- doSkySource : `bool` Value of the config flag determining whether to insert sky sources. """ charImConfig = CharacterizeImageConfig() charImConfig.measurePsf.psfDeterminer = 'piff' charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0 charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibConfig.doSkySources = doSkySources calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) if doSkySources: self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames()) else: self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2") # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2") calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] isrResult1 = IsrTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get( "postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) icResult1 = CharacterizeImageTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure, exposureIdInfo=exposureIdInfo) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get( "icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background) calResult1 = CalibrateTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat, exposureIdInfo=exposureIdInfo, ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get( "calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual(calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent")) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, display=False, verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() calibrateTask = CalibrateTask(config=config.calibrate) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement, schema=schema, algMetadata=algMetadata) exposureDict = {}; calibSourcesDict = {}; sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance, verbose) # # process the data # calibSources = None # sources used to calibrate the frame (photom, astrom, psf) if config.doCalibrate: result = calibrateTask.run(exposure) exposure, sources = result.exposure, result.sources if returnCalibSources: calibSources = sources else: if not exposure.getPsf(): calibrateTask.installInitialPsf(exposure) if config.edgeRolloff.applyModel: if verbose: print "Adding edge rolloff distortion to the exposure WCS" addEdgeRolloffDistortion(exposure, config.edgeRolloff) exposureDict[inputFile] = exposure calibSourcesDict[inputFile] = calibSources result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources, exposure.getPsf()) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) if display: # display on ds9 (see also --debug argparse option) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = None frame = 1 ds9.mtv(exposure, title=os.path.split(inputFile)[1], frame=frame) with ds9.Buffering(): for s in sources: xy = s.getCentroid() ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame) ds9.dot(s.getShape(), *xy, ctype=ds9.RED, frame=frame) if radii: for radius in radii: ds9.dot('o', *xy, size=radius, ctype=ds9.YELLOW, frame=frame) return exposureDict, calibSourcesDict, sourcesDict
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask( config=getObsTestConfig(IsrTask), name="isr2" ) # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2" ) calibrateTask = CalibrateTask( config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema ) try: dataId = dict(visit=1) dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()] isrResult1 = IsrTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage() ) icResult1 = CharacterizeImageTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background ) calResult1 = CalibrateTask.parseAndRun( args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage() ) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat ) self.assertCatalogsEqual( calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent") ) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background ) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background ) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, displayResults=[], verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() isrTask = IsrTask(config=config.isr) calibrateTask = CalibrateTask(config=config.calibrate) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement, schema=schema, algMetadata=algMetadata) sourceMeasurementTask.config.doApplyApCorr = 'yes' # # Add fields needed to identify stars while calibrating # keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected", "Source was detected by calibrate")), None)] for key in calibrateTask.getCalibKeys(): keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key)) exposureDict = {}; calibSourcesDict = {}; sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance) # if config.interpPlanes: import lsst.ip.isr as ipIsr defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes, growFootprints=0) isrTask.run(exposure, defects=defects) # # process the data # if config.doCalibrate: result = calibrateTask.run(exposure) exposure, calibSources = result.exposure, result.sources else: calibSources = None if not exposure.getPsf(): calibrateTask.installInitialPsf(exposure) exposureDict[inputFile] = exposure calibSourcesDict[inputFile] = calibSources if returnCalibSources else None result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources, exposure.getPsf()) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) propagateCalibFlags(keysToCopy, calibSources, sources) if displayResults: # display results of processing (see also --debug argparse option) showApertures = "showApertures".upper() in displayResults showShapes = "showShapes".upper() in displayResults display = afwDisplay.getDisplay(frame=1) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = [] display.mtv(exposure, title=os.path.split(inputFile)[1]) with display.Buffering(): for s in sources: xy = s.getCentroid() display.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) if showShapes: display.dot(s.getShape(), *xy, ctype=afwDisplay.RED) if showApertures: for radius in radii: display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW) return exposureDict, calibSourcesDict, sourcesDict
algMetaData = dafBase.PropertyList() measureTask = SourceMeasurementTask(schema, algMetaData, config=meas_config) exposure = afwImage.ExposureF(image_file) # # Add variance, inferred from the data, to the exposure # mi = exposure.getMaskedImage() varclip = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue() mi.getVariance().getArray()[:] = np.ones(mi.getArrays()[0].shape)*varclip # # Run calibration task # result = calibrateTask.run(exposure) exposure, calibSources = result.exposure, result.sources # # Create XYTransform for the edge rolloff effect # warper = EdgeRolloffWarper(2, 30, 400, 2, 30, 400) transform = warper.transform.clone() distortedTanWcs = afwImage.DistortedTanWcs(afwImage.TanWcs.cast(exposure.getWcs()), transform) #output_table = afwTable.SourceTable.make(schema) #sources = detectionTask.run(output_table, exposure, sigma=5).sources # ## Disable psfFlux since it is crashing. #measureTask.config.slots.psfFlux = None #measureTask.run(exposure, calibSources)