def processCCDs(image): from lsst.pipe.tasks.calibrate import CalibrateTask, CalibrateConfig from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask calibRes = None # init tasks charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False, doDeblend=False) calibrateTask = CalibrateTask(config=calibrateConfig) try: # characterize image charRes = charImage.characterize(image, exposureIdInfo=None, background=None) # calibrate image calibRes = calibrateTask.calibrate(charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except Exception as e: print "failed to calibrate the image" print str(e) return calibRes
def run(display=False): # # Create the tasks # charImageConfig = CharacterizeImageTask.ConfigClass() charImageTask = CharacterizeImageTask(config=charImageConfig) config = CalibrateTask.ConfigClass() config.astrometry.retarget(MyAstrometryTask) calibrateTask = CalibrateTask(config=config) # load the data # Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo, # but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs). exposure = loadData() exposureIdInfo = ExposureIdInfo(expId=1, expBits=5) # characterize the exposure to repair cosmic rays and fit a PSF model # display now because CalibrateTask modifies the exposure in place charRes = charImageTask.characterize(exposure=exposure, exposureIdInfo=exposureIdInfo) if display: displayFunc(charRes.exposure, charRes.sourceCat, frame=1) # calibrate the exposure calRes = calibrateTask.calibrate(exposure=charRes.exposure, exposureIdInfo=exposureIdInfo) if display: displayFunc(calRes.exposure, calRes.sourceCat, frame=2)
def setUp(self): # Load sample input from disk expPath = os.path.join(getPackageDir("pipe_tasks"), "tests", "data", "v695833-e0-c000-a00.sci.fits") self.exposure = afwImage.ExposureF(expPath) # Characterize the image (create PSF, etc.) charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) self.charImResults = charImTask.run(self.exposure) # set log level so that warnings do not display Log.getLogger("calibrate").setLevel(Log.ERROR)
def run(self, sensor_id, infile, gains, bias_frame=None): # # Process a CCD image mosaic # if self.config.verbose: self.log.info("processing {0}".format(infile)) image = make_ccd_mosaic(infile, bias_frame=bias_frame, gains=gains) exposure = afwImage.ExposureF(image.getBBox()) exposure.setImage(image) # # Set up characterize task configuration # nsig = self.config.nsig bgbinsize = self.config.bgbinsize minpixels = self.config.minpixels charConfig = CharacterizeImageConfig() charConfig.doMeasurePsf = False charConfig.doApCorr = False charConfig.repair.doCosmicRay = False charConfig.detection.minPixels = minpixels charConfig.detection.background.binSize = bgbinsize charConfig.detection.thresholdType = "stdev" charConfig.detection.thresholdValue = nsig hsm_plugins = set(["ext_shapeHSM_HsmShapeBj", "ext_shapeHSM_HsmShapeLinear", "ext_shapeHSM_HsmShapeKsb", "ext_shapeHSM_HsmShapeRegauss", "ext_shapeHSM_HsmSourceMoments", "ext_shapeHSM_HsmPsfMoments"]) charConfig.measurement.plugins.names |= hsm_plugins charTask = CharacterizeImageTask(config=charConfig) if lsst.pipe.tasks.__version__.startswith('17.0'): result = charTask.run(exposure) else: result = charTask.characterize(exposure) src = result.sourceCat if self.config.verbose: self.log.info("Detected {0} objects".format(len(src))) # # Save catalog results to file # output_dir = self.config.output_dir if self.config.output_file is None: output_file = os.path.join(output_dir, '{0}_source_catalog.cat'.format(sensor_id)) else: output_file = os.path.join(output_dir, self.config.output_file) if self.config.verbose: self.log.info("Writing spot results file to {0}".format(output_file)) src.writeFits(output_file)
def testFlags(self): # Test that all of the flags are defined and there is no reservation by default # also test that used sources are a subset of candidate sources task = CharacterizeImageTask() results = task.characterize(self.exposure) used = 0 reserved = 0 for source in results.sourceCat: if source.get("calib_psfUsed"): used += 1 self.assertTrue(source.get("calib_psfCandidate")) if source.get("calib_psf_reserved"): reserved += 1 self.assertGreater(used, 0) self.assertEqual(reserved, 0)
def testFlags(self): # Test that all of the flags are defined and there is no reservation by default # also test that used sources are a subset of candidate sources config = CharacterizeImageConfig() config.measurePsf.psfDeterminer = 'piff' config.measurePsf.psfDeterminer['piff'].spatialOrder = 0 task = CharacterizeImageTask(config=config) results = task.run(self.exposure) used = 0 reserved = 0 for source in results.sourceCat: if source.get("calib_psf_used"): used += 1 self.assertTrue(source.get("calib_psf_candidate")) if source.get("calib_psf_reserved"): reserved += 1 self.assertGreater(used, 0) self.assertEqual(reserved, 0)
def execute(self, dataRef): """!Characterize a science image @param dataRef: butler data reference @return a pipeBase Struct containing the results """ self.log.info("Performing Super CharacterizeImage on sensor data ID %s" % (dataRef.dataId,)) self.log.info("Reading input data using dataRef") inputData = self.read_input_data(dataRef) self.log.info("Running operations. The run() method should not take anything Butler") result = CharacterizeImageTask.characterize(CharacterizeImageTask(config=self.config, log=self.log, schema=SourceTable.makeMinimalSchema()), **inputData.getDict()) self.log.info("Writing output data using dataRef") self.write_output_data(dataRef, result) return result
def testIsPrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0)
def _checkSkySourceColumnExistence(self, doSkySources): """Implements sky_source column checking. Parameters ---------- doSkySource : `bool` Value of the config flag determining whether to insert sky sources. """ charImConfig = CharacterizeImageConfig() charImConfig.measurePsf.psfDeterminer = 'piff' charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0 charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibConfig.doSkySources = doSkySources calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) if doSkySources: self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames()) else: self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def testComponents(self): """Test that we can run the first-level subtasks of ProcessCcdTasks. This tests that we can run these subtasks from the command-line independently (they're all CmdLineTasks) as well as directly from Python (without giving them access to a Butler). Aside from verifying that no exceptions are raised, we simply tests that most persisted results are present and equivalent to both in-memory results. """ outPath = tempfile.mkdtemp( ) if OutputName is None else "{}-Components".format(OutputName) # We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it # to those tasks. inputButler = lsst.daf.persistence.Butler(InputDir) # Construct task instances we can use directly from Python isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2") # If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these # tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class # that doesn't require a Butler. If we don't, we should construct a butler-based on outside these # task constructors and pass the LoadReferenceObjectsTask instance to the task constructors. charImageTask = CharacterizeImageTask( config=getObsTestConfig(CharacterizeImageTask), name="charImage2") calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask), name="calibrate2", icSourceSchema=charImageTask.schema) try: dataId = dict(visit=1) dataIdStrList = [ "%s=%s" % (key, val) for key, val in dataId.items() ] isrResult1 = IsrTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) # We'll just use the butler to get the original image and calibration frames; it's not clear # extending the test coverage to include that is worth it. dataRef = inputButler.dataRef("raw", dataId=dataId) rawExposure = dataRef.get("raw", immediate=True) camera = dataRef.get("camera") isrData = isrTask.readIsrData(dataRef, rawExposure) exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId) isrResult2 = isrTask.run( rawExposure, bias=isrData.bias, linearizer=isrData.linearizer, flat=isrData.flat, defects=isrData.defects, fringes=isrData.fringes, bfKernel=isrData.bfKernel, camera=camera, ) self.assertMaskedImagesEqual( isrResult1.parsedCmd.butler.get( "postISRCCD", dataId, immediate=True).getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( isrResult2.exposure.getMaskedImage(), isrResult1.resultList[0].result.exposure.getMaskedImage()) icResult1 = CharacterizeImageTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) icResult2 = charImageTask.run(isrResult2.exposure, exposureIdInfo=exposureIdInfo) self.assertMaskedImagesEqual( icResult1.parsedCmd.butler.get( "icExp", dataId, immediate=True).getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( icResult2.exposure.getMaskedImage(), icResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True), icResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual( icResult2.sourceCat, icResult1.resultList[0].result.sourceCat, ) self.assertBackgroundListsEqual( icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True), icResult1.resultList[0].result.background) self.assertBackgroundListsEqual( icResult2.background, icResult1.resultList[0].result.background) calResult1 = CalibrateTask.parseAndRun( args=[ InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id" ] + dataIdStrList, doReturnResults=True, ) calResult2 = calibrateTask.run( icResult2.exposure, background=icResult2.background, icSourceCat=icResult2.sourceCat, exposureIdInfo=exposureIdInfo, ) self.assertMaskedImagesEqual( calResult1.parsedCmd.butler.get( "calexp", dataId, immediate=True).getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertMaskedImagesEqual( calResult2.exposure.getMaskedImage(), calResult1.resultList[0].result.exposure.getMaskedImage()) self.assertCatalogsEqual( calResult1.parsedCmd.butler.get("src", dataId, immediate=True), calResult1.resultList[0].result.sourceCat) self.assertCatalogsEqual(calResult2.sourceCat, calResult1.resultList[0].result.sourceCat, skipCols=("id", "parent")) self.assertBackgroundListsEqual( calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True), calResult1.resultList[0].result.background) self.assertBackgroundListsEqual( calResult2.background, calResult1.resultList[0].result.background) finally: if OutputName is None: shutil.rmtree(outPath) else: print("testProcessCcd.py's output data saved to %r" % (OutputName, ))
######################################## ####### Setup LSST begins ######################################## visit=289697 ccdnum = 1 HOME_PATH = os.path.expanduser("~") butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw') exposure = butler.get("instcal", visit=visit, ccdnum=ccdnum, filter='g', immediate=True) mask = exposure.getMaskedImage().getMask().getArray() #TODO: move this matrix to SciDB variance = exposure.getMaskedImage().getVariance().getArray() #TODO: move this matrix to SciDB butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw') filename = HOME_PATH + "/lsst_data/raw/crblasted0289697/instcal0289697.1.fits" fitsHeader = afwImage.readMetadata(filename) wcs = afwImage.makeWcs(fitsHeader) charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] sys.stderr.write("=====> DFZ 3/24/2017: mask.shape = " + str(mask.shape) + "\n") ######################################## ####### Setup LSST ends ########################################
def main(): # try out one exposure #visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"] visits = ["0288976", "0288935"] ccds = [] exit(0) for i in range(1, 61): ccds.append(i) filterName = 'g' DATA_PATH = "/root/extra_home/lsst_data/" #spathprefix = "/home/dongfang/download/lsst_data/" spathprefix = DATA_PATH + "raw/" #calexpsloc = "/home/dongfang/download/lsst_data/calexps/" calexpsloc = DATA_PATH + "calexps/" #coaddloc = "/home/dongfang/download/lsst_data/coadds/" coaddloc = DATA_PATH + "coadds/" #mergecoaddloc = "/home/dongfang/download/lsst_data/merge/" mergecoaddloc = DATA_PATH + "merge/" # Characterize Image charImageConfig = CharacterizeImageConfig() charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig( projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] coaddTempDict = {} calibResDict = {} f = open("log.txt", 'wb') start = datetime.datetime.now() #process CCDs to create calexps. for v in visits: for ccd in ccds: visit = int(v) filename = "instcal" + v + "." + str(ccd) + ".fits" calexpfn = calexpsloc + v + "/" + filename source = spathprefix + v + "/" + filename exposure = afwImg.ExposureF(source) try: # Characterize Image charRes = charImage.characterize(exposure, exposureIdInfo=None, background=None) except: f.write("DFZ DEBUG at charRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: # Caliberate Image calibRes = calibrateTask.calibrate( charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except: f.write("DFZ DEBUG at calibRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: #write out calexps calibRes.exposure.writeFits(calexpfn) #calbresDict.append((v,ccd),calibRes) except: f.write("DFZ DEBUG at calibRes.exposure: errors in visit " + v + ", ccd " + str(ccd) + "\n") end = datetime.datetime.now() d = end - start f.write("time for creating calexps: ") f.write(str(d.total_seconds())) f.write("\n") #time for creating co-add tempexps. start = datetime.datetime.now() # map calexps to patch-ids visit = visits[0] ccdsPerPatch = [] for ccd in ccds: filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename exposure = afwImg.ExposureF(source) bbox = exposure.getBBox() wcs = exposure.getWcs() corners = bbox.getCorners() xIndexMax, yIndexMax = tract.findPatch( wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex() xIndexMin, yIndexMin = tract.findPatch( wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex() yy = range(yIndexMin, yIndexMax + 1) xx = range(xIndexMin, xIndexMax + 1) for yIdx in yy: for xIdx in xx: ccdsPerPatch.append((ccd, (xIdx, yIdx))) print len(ccdsPerPatch) #import cPickle #cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch) # import cPickle # f = open("ccdsInPatch.p",'wb') # cPickle.dump(ccdsInPatch,f) #import cPickle #ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb')) df = pd.DataFrame(ccdsPerPatch) dfgby = df.groupby(1) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) coaddTempExpDict = {} for visit in visits: for a in dfgby.indices: coaddTempExpDict[a] = {} xInd = a[0] yInd = a[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set( numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby.get_group(a)[0].ravel(): print a print b if not setInputRecorder: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) try: inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) except: f.write("DFZ DEBUG at inputRecorder\n") setInputRecorder = True numGoodPix = 0 ccd = b filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename calExp = afwImg.ExposureF(source) ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf( modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) coaddTempExpDict[a][v] = coaddTempExp coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str( xInd) + "_" + str(yInd) + ".fits" coaddTempExp.writeFits(coaddfilename) end = datetime.datetime.now() d = end - start f.write("time for creating co-add tempexps:\n ") f.write(str(d.total_seconds())) f.write("\n") #DFZ: stop here exit(0) start = datetime.datetime.now() config = AssembleCoaddConfig() assembleTask = AssembleCoaddTask(config=config) mergcoadds = {} for a in dfgby.indices: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) xInd = a[0] yInd = a[1] imageScalerRes = prepareInputs(coaddTempExpDict[a].values(), coaddTempExpDict[a].keys(), assembleTask) mask = None doClip = False if mask is None: mask = assembleTask.getBadPixelMask() statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip) statsCtrl.setNumIter(assembleTask.config.clipIter) statsCtrl.setAndMask(mask) statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) for plane, threshold in assembleTask.config.maskPropagationThresholds.items( ): bit = afwImage.MaskU.getMaskPlane(plane) statsCtrl.setMaskPropagationThreshold(bit, threshold) if doClip: statsFlags = afwMath.MEANCLIP else: statsFlags = afwMath.MEAN coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib()) coaddExposure.getInfo().setCoaddInputs( assembleTask.inputRecorder.makeCoaddInputs()) #remember to set metadata if you want any hope of running detection and measurement on this coadd: #self.assembleMetadata(coaddExposure, tempExpRefList, weightList) #most important thing is the psf coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter()) coaddInputs = coaddExposure.getInfo().getCoaddInputs() for tempExp, weight in zip(coaddTempExpDict[a].values(), imageScalerRes.weightList): assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) #takes numCcds as argument coaddInputs.ccds.reserve(ccdsinPatch) coaddInputs.visits.reserve(len(imageScalerRes.dataIdList)) psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs()) coaddExposure.setPsf(psf) maskedImageList = afwImage.vectorMaskedImageF() coaddMaskedImage = coaddExposure.getMaskedImage() for dataId, imageScaler, exposure in zip( imageScalerRes.dataIdList, imageScalerRes.imageScalerList, coaddTempExpDict[a].values()): print dataId, imageScaler, exposure maskedImage = exposure.getMaskedImage() imageScaler.scaleMaskedImage(maskedImage) maskedImageList.append(maskedImage) maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, imageScalerRes.weightList) coaddMaskedImage.assign(maskedImage, skyInfo.bbox) coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) # write out Coadd! mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits" mergcoadds[a] = coaddExposure coaddExposure.writeFits(mergefilename) end = datetime.datetime.now() d = end - start f.write("time for creating merged co-adds:\n ") f.write(str(d.total_seconds())) f.write("\n") start = datetime.datetime.now() config = DetectCoaddSourcesConfig() detectCoaddSources = DetectCoaddSourcesTask(config=config) for a in dfgby.indices: # Detect on Coadd exp = mergcoadds[a] detRes = detectCoaddSources.runDetection(exp, idFactory=None) end = datetime.datetime.now() d = end - start f.write("time for detecting sources:\n ") f.write(str(d.total_seconds())) f.close()
def setupCharTask(self): ''' set up the CharacterizeImageTask task ''' return CharacterizeImageTask(config=self.config)
def run(config, inputFiles, weightFiles=None, varianceFiles=None, returnCalibSources=False, displayResults=[], verbose=False): # # Create the tasks # schema = afwTable.SourceTable.makeMinimalSchema() algMetadata = dafBase.PropertyList() isrTask = IsrTask(config=config.isr) charImageTask = CharacterizeImageTask(None, config=config.charImage) sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema) if config.doDeblend: if SourceDeblendTask: sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema) else: print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False" config.doDeblend = False sourceMeasurementTask = SingleFrameMeasurementTask( schema=schema, config=config.measurement, algMetadata=algMetadata) keysToCopy = [] for key in [ charImageTask.measurePsf.reservedKey, charImageTask.measurePsf.usedKey, ]: keysToCopy.append( (schema.addField(charImageTask.schema.find(key).field), key)) exposureDict = {} calibSourcesDict = {} sourcesDict = {} for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles): # # Create the output table # tab = afwTable.SourceTable.make(schema) # # read the data # if verbose: print "Reading %s" % inputFile exposure = makeExposure(inputFile, weightFile, varianceFile, config.badPixelValue, config.variance) # if config.interpPlanes: import lsst.ip.isr as ipIsr defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes, growFootprints=0) isrTask.run(exposure, defects=defects) # # process the data # if config.doCalibrate: result = charImageTask.characterize(exposure) exposure, calibSources = result.exposure, result.sourceCat else: calibSources = None if not exposure.getPsf(): charImageTask.installSimplePsf.run(exposure) exposureDict[inputFile] = exposure calibSourcesDict[ inputFile] = calibSources if returnCalibSources else None result = sourceDetectionTask.run(tab, exposure) sources = result.sources sourcesDict[inputFile] = sources if config.doDeblend: sourceDeblendTask.run(exposure, sources) sourceMeasurementTask.measure(exposure, sources) if verbose: print "Detected %d objects" % len(sources) propagatePsfFlags(keysToCopy, calibSources, sources) if displayResults: # display results of processing (see also --debug argparse option) showApertures = "showApertures".upper() in displayResults showPSFs = "showPSFs".upper() in displayResults showShapes = "showShapes".upper() in displayResults display = afwDisplay.getDisplay(frame=1) if algMetadata.exists("base_CircularApertureFlux_radii"): radii = algMetadata.get("base_CircularApertureFlux_radii") else: radii = [] display.mtv(exposure, title=os.path.split(inputFile)[1]) with display.Buffering(): for s in sources: xy = s.getCentroid() display.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN) if showPSFs and (s.get("calib_psfUsed") or s.get("calib_psfReserved")): display.dot( 'o', *xy, size=10, ctype=afwDisplay.GREEN if s.get("calib_psfUsed") else afwDisplay.YELLOW) if showShapes: display.dot(s.getShape(), *xy, ctype=afwDisplay.RED) if showApertures: for radius in radii: display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW) return exposureDict, calibSourcesDict, sourcesDict
if spots_bbox.overlaps(amp.getBBox()): gain = gain_data[amp.getName()] img = exposure_copy.image sim = img.Factory(img, amp.getBBox()) sim *= gain print(amp.getName(), gain, amp.getBBox()) sys.stdout.flush() if do_bf_corr: brighterFatterCorrection(exposure_copy[amp.getBBox()],bf_kernel.kernel[amp.getName()],20,10,False) else: continue # Now trim the exposure down to the region of interest trimmed_exposure = exposure_copy[spots_bbox] # Now find and characterize the spots charTask = CharacterizeImageTask(config=charConfig) tstart=time.time() charResult = charTask.run(trimmed_exposure) spotCatalog = charResult.sourceCat print("%s, Correction = %r, Characterization took "%(amp.getName(),do_bf_corr),str(time.time()-tstart)[:4]," seconds") sys.stdout.flush() # Now trim out spots not between minSize and maxSize select = ((spotCatalog['base_SdssShape_xx'] >= minSize) & (spotCatalog['base_SdssShape_xx'] <= maxSize) & (spotCatalog['base_SdssShape_yy'] >= minSize) & (spotCatalog['base_SdssShape_yy'] <= maxSize)) spotCatalog = spotCatalog.subset(select) x2 = spotCatalog['base_SdssShape_xx'] y2 = spotCatalog['base_SdssShape_yy'] flux = spotCatalog['base_SdssShape_instFlux'] numspots = len(flux) print("Detected ",len(spotCatalog)," objects, Flux = %f, X2 = %f, Y2 = %f"%(np.nanmean(flux),np.nanmean(x2),np.nanmean(y2))) sys.stdout.flush()
def run(self, ccdExposure, *, camera=None, bias=None, dark=None, flat=None, defects=None, linearizer=None, crosstalk=None, bfKernel=None, bfGains=None, ptc=None, crosstalkSources=None, isrBaseConfig=None): """Run isr and cosmic ray repair using, doing as much isr as possible. Retrieves as many calibration products as are available, and runs isr with those settings enabled, but always returns an assembled image at a minimum. Then performs cosmic ray repair if configured to. Parameters ---------- ccdExposure : `lsst.afw.image.Exposure` The raw exposure that is to be run through ISR. The exposure is modified by this method. camera : `lsst.afw.cameraGeom.Camera`, optional The camera geometry for this exposure. Required if one or more of ``ccdExposure``, ``bias``, ``dark``, or ``flat`` does not have an associated detector. bias : `lsst.afw.image.Exposure`, optional Bias calibration frame. linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional Functor for linearization. crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional Calibration for crosstalk. crosstalkSources : `list`, optional List of possible crosstalk sources. dark : `lsst.afw.image.Exposure`, optional Dark calibration frame. flat : `lsst.afw.image.Exposure`, optional Flat calibration frame. ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional Photon transfer curve dataset, with, e.g., gains and read noise. bfKernel : `numpy.ndarray`, optional Brighter-fatter kernel. bfGains : `dict` of `float`, optional Gains used to override the detector's nominal gains for the brighter-fatter correction. A dict keyed by amplifier name for the detector in question. defects : `lsst.ip.isr.Defects`, optional List of defects. fringes : `lsst.pipe.base.Struct`, optional Struct containing the fringe correction data, with elements: - ``fringes``: fringe calibration frame (`afw.image.Exposure`) - ``seed``: random seed derived from the ccdExposureId for random number generator (`uint32`) opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional A ``TransmissionCurve`` that represents the throughput of the, optics, to be evaluated in focal-plane coordinates. filterTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the filter itself, to be evaluated in focal-plane coordinates. sensorTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the sensor itself, to be evaluated in post-assembly trimmed detector coordinates. atmosphereTransmission : `lsst.afw.image.TransmissionCurve` A ``TransmissionCurve`` that represents the throughput of the atmosphere, assumed to be spatially constant. detectorNum : `int`, optional The integer number for the detector to process. strayLightData : `object`, optional Opaque object containing calibration information for stray-light correction. If `None`, no correction will be performed. illumMaskedImage : `lsst.afw.image.MaskedImage`, optional Illumination correction image. isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional An isrTask config to act as the base configuration. Options which involve applying a calibration product are ignored, but this allows for the configuration of e.g. the number of overscan columns. Returns ------- result : `lsst.pipe.base.Struct` Result struct with component: - ``exposure`` : `afw.image.Exposure` The ISRed and cosmic-ray-repaired exposure. """ isrConfig = isrBaseConfig if isrBaseConfig else IsrTask.ConfigClass() isrConfig.doBias = False isrConfig.doDark = False isrConfig.doFlat = False isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doLinearize = False isrConfig.doCrosstalk = False isrConfig.doBrighterFatter = False isrConfig.usePtcGains = False if bias: isrConfig.doBias = True self.log.info("Running with bias correction") if dark: isrConfig.doDark = True self.log.info("Running with dark correction") if flat: isrConfig.doFlat = True self.log.info("Running with flat correction") # TODO: deal with fringes here if defects: isrConfig.doDefect = True self.log.info("Running with defect correction") if linearizer: isrConfig.doLinearize = True self.log.info("Running with linearity correction") if crosstalk: isrConfig.doCrosstalk = True self.log.info("Running with crosstalk correction") if bfKernel: isrConfig.doBrighterFatter = True self.log.info("Running with brighter-fatter correction") if ptc: isrConfig.usePtcGains = True self.log.info("Running with ptc correction") isrConfig.doWrite = False isrTask = IsrTask(config=isrConfig) result = isrTask.run( ccdExposure, camera=camera, bias=bias, dark=dark, flat=flat, # fringes=pipeBase.Struct(fringes=None), defects=defects, linearizer=linearizer, crosstalk=crosstalk, bfKernel=bfKernel, bfGains=bfGains, ptc=ptc, crosstalkSources=crosstalkSources, isGen3=True, ) postIsr = result.exposure if self.config.doRepairCosmics: try: # can fail due to too many CRs detected, and we always want an exposure back self.log.info("Repairing cosmics...") if postIsr.getPsf() is None: installPsfTask = InstallGaussianPsfTask() installPsfTask.run(postIsr) # TODO: try adding a reasonably wide Gaussian as a temp PSF # and then just running repairTask on its own without any # imChar. It should work, and be faster. repairConfig = CharacterizeImageTask.ConfigClass() repairConfig.doMeasurePsf = False repairConfig.doApCorr = False repairConfig.doDeblend = False repairConfig.doWrite = False repairConfig.repair.cosmicray.nCrPixelMax = 200000 repairTask = CharacterizeImageTask(config=repairConfig) repairTask.repair.run(postIsr) except Exception as e: self.log.warning(f"During CR repair caught: {e}") # exposure is returned for convenience to mimic isrTask's API. return pipeBase.Struct(exposure=postIsr, outputExposure=postIsr)