def setupCharConfig(self, doCmodel=False): ''' create the config for CaracterizeImageTask. Set up appropriate configurations for galsim data ''' charConfig = CharacterizeImageConfig() # switch these off for simulated data charConfig.doMeasurePsf = False charConfig.doApCorr = False charConfig.repair.doCosmicRay = False charConfig.doDeblend = True # threshold detection charConfig.detection.thresholdValue = self.threshold charConfig.detection.includeThresholdMultiplier = 1.0 charConfig.detection.minPixels = 1 # these are parameters Lee set charConfig.installSimplePsf.fwhm = 5 charConfig.installSimplePsf.width = 55 if self.doCModel: charConfig.measurement.plugins.names |= ["ext_shapeHSM_HsmSourceMoments"] charConfig.measurement.plugins.names |= ["modelfit_DoubleShapeletPsfApprox", "modelfit_CModel"] charConfig.measurement.slots.modelFlux = 'modelfit_CModel' return charConfig
def run(self, sensor_id, infile, gains, bias_frame=None): # # Process a CCD image mosaic # if self.config.verbose: self.log.info("processing {0}".format(infile)) image = make_ccd_mosaic(infile, bias_frame=bias_frame, gains=gains) exposure = afwImage.ExposureF(image.getBBox()) exposure.setImage(image) # # Set up characterize task configuration # nsig = self.config.nsig bgbinsize = self.config.bgbinsize minpixels = self.config.minpixels charConfig = CharacterizeImageConfig() charConfig.doMeasurePsf = False charConfig.doApCorr = False charConfig.repair.doCosmicRay = False charConfig.detection.minPixels = minpixels charConfig.detection.background.binSize = bgbinsize charConfig.detection.thresholdType = "stdev" charConfig.detection.thresholdValue = nsig hsm_plugins = set(["ext_shapeHSM_HsmShapeBj", "ext_shapeHSM_HsmShapeLinear", "ext_shapeHSM_HsmShapeKsb", "ext_shapeHSM_HsmShapeRegauss", "ext_shapeHSM_HsmSourceMoments", "ext_shapeHSM_HsmPsfMoments"]) charConfig.measurement.plugins.names |= hsm_plugins charTask = CharacterizeImageTask(config=charConfig) if lsst.pipe.tasks.__version__.startswith('17.0'): result = charTask.run(exposure) else: result = charTask.characterize(exposure) src = result.sourceCat if self.config.verbose: self.log.info("Detected {0} objects".format(len(src))) # # Save catalog results to file # output_dir = self.config.output_dir if self.config.output_file is None: output_file = os.path.join(output_dir, '{0}_source_catalog.cat'.format(sensor_id)) else: output_file = os.path.join(output_dir, self.config.output_file) if self.config.verbose: self.log.info("Writing spot results file to {0}".format(output_file)) src.writeFits(output_file)
def setUp(self): # Load sample input from disk expPath = os.path.join(getPackageDir("pipe_tasks"), "tests", "data", "v695833-e0-c000-a00.sci.fits") self.exposure = afwImage.ExposureF(expPath) # Characterize the image (create PSF, etc.) charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) self.charImResults = charImTask.run(self.exposure) # set log level so that warnings do not display Log.getLogger("calibrate").setLevel(Log.ERROR)
def testFlags(self): # Test that all of the flags are defined and there is no reservation by default # also test that used sources are a subset of candidate sources config = CharacterizeImageConfig() config.measurePsf.psfDeterminer = 'piff' config.measurePsf.psfDeterminer['piff'].spatialOrder = 0 task = CharacterizeImageTask(config=config) results = task.run(self.exposure) used = 0 reserved = 0 for source in results.sourceCat: if source.get("calib_psf_used"): used += 1 self.assertTrue(source.get("calib_psf_candidate")) if source.get("calib_psf_reserved"): reserved += 1 self.assertGreater(used, 0) self.assertEqual(reserved, 0)
def testIsPrimaryFlag(self): """Tests detect_isPrimary column gets added when run, and that sources labelled as detect_isPrimary are not sky sources and have no children. """ charImConfig = CharacterizeImageConfig() charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) outputCat = calibResults.outputCat self.assertTrue("detect_isPrimary" in outputCat.schema.getNames()) # make sure all sky sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])), 0) # make sure all parent sources are flagged as not primary self.assertEqual( sum((outputCat["detect_isPrimary"]) & (outputCat["deblend_nChild"] > 0)), 0)
def _checkSkySourceColumnExistence(self, doSkySources): """Implements sky_source column checking. Parameters ---------- doSkySource : `bool` Value of the config flag determining whether to insert sky sources. """ charImConfig = CharacterizeImageConfig() charImConfig.measurePsf.psfDeterminer = 'piff' charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0 charImTask = CharacterizeImageTask(config=charImConfig) charImResults = charImTask.run(self.exposure) calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibConfig.doSkySources = doSkySources calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) if doSkySources: self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames()) else: self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def main(): # try out one exposure #visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"] visits = ["0288976", "0288935"] ccds = [] exit(0) for i in range(1, 61): ccds.append(i) filterName = 'g' DATA_PATH = "/root/extra_home/lsst_data/" #spathprefix = "/home/dongfang/download/lsst_data/" spathprefix = DATA_PATH + "raw/" #calexpsloc = "/home/dongfang/download/lsst_data/calexps/" calexpsloc = DATA_PATH + "calexps/" #coaddloc = "/home/dongfang/download/lsst_data/coadds/" coaddloc = DATA_PATH + "coadds/" #mergecoaddloc = "/home/dongfang/download/lsst_data/merge/" mergecoaddloc = DATA_PATH + "merge/" # Characterize Image charImageConfig = CharacterizeImageConfig() charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig( projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] coaddTempDict = {} calibResDict = {} f = open("log.txt", 'wb') start = datetime.datetime.now() #process CCDs to create calexps. for v in visits: for ccd in ccds: visit = int(v) filename = "instcal" + v + "." + str(ccd) + ".fits" calexpfn = calexpsloc + v + "/" + filename source = spathprefix + v + "/" + filename exposure = afwImg.ExposureF(source) try: # Characterize Image charRes = charImage.characterize(exposure, exposureIdInfo=None, background=None) except: f.write("DFZ DEBUG at charRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: # Caliberate Image calibRes = calibrateTask.calibrate( charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except: f.write("DFZ DEBUG at calibRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: #write out calexps calibRes.exposure.writeFits(calexpfn) #calbresDict.append((v,ccd),calibRes) except: f.write("DFZ DEBUG at calibRes.exposure: errors in visit " + v + ", ccd " + str(ccd) + "\n") end = datetime.datetime.now() d = end - start f.write("time for creating calexps: ") f.write(str(d.total_seconds())) f.write("\n") #time for creating co-add tempexps. start = datetime.datetime.now() # map calexps to patch-ids visit = visits[0] ccdsPerPatch = [] for ccd in ccds: filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename exposure = afwImg.ExposureF(source) bbox = exposure.getBBox() wcs = exposure.getWcs() corners = bbox.getCorners() xIndexMax, yIndexMax = tract.findPatch( wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex() xIndexMin, yIndexMin = tract.findPatch( wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex() yy = range(yIndexMin, yIndexMax + 1) xx = range(xIndexMin, xIndexMax + 1) for yIdx in yy: for xIdx in xx: ccdsPerPatch.append((ccd, (xIdx, yIdx))) print len(ccdsPerPatch) #import cPickle #cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch) # import cPickle # f = open("ccdsInPatch.p",'wb') # cPickle.dump(ccdsInPatch,f) #import cPickle #ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb')) df = pd.DataFrame(ccdsPerPatch) dfgby = df.groupby(1) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) coaddTempExpDict = {} for visit in visits: for a in dfgby.indices: coaddTempExpDict[a] = {} xInd = a[0] yInd = a[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set( numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby.get_group(a)[0].ravel(): print a print b if not setInputRecorder: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) try: inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) except: f.write("DFZ DEBUG at inputRecorder\n") setInputRecorder = True numGoodPix = 0 ccd = b filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename calExp = afwImg.ExposureF(source) ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf( modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) coaddTempExpDict[a][v] = coaddTempExp coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str( xInd) + "_" + str(yInd) + ".fits" coaddTempExp.writeFits(coaddfilename) end = datetime.datetime.now() d = end - start f.write("time for creating co-add tempexps:\n ") f.write(str(d.total_seconds())) f.write("\n") #DFZ: stop here exit(0) start = datetime.datetime.now() config = AssembleCoaddConfig() assembleTask = AssembleCoaddTask(config=config) mergcoadds = {} for a in dfgby.indices: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) xInd = a[0] yInd = a[1] imageScalerRes = prepareInputs(coaddTempExpDict[a].values(), coaddTempExpDict[a].keys(), assembleTask) mask = None doClip = False if mask is None: mask = assembleTask.getBadPixelMask() statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip) statsCtrl.setNumIter(assembleTask.config.clipIter) statsCtrl.setAndMask(mask) statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) for plane, threshold in assembleTask.config.maskPropagationThresholds.items( ): bit = afwImage.MaskU.getMaskPlane(plane) statsCtrl.setMaskPropagationThreshold(bit, threshold) if doClip: statsFlags = afwMath.MEANCLIP else: statsFlags = afwMath.MEAN coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib()) coaddExposure.getInfo().setCoaddInputs( assembleTask.inputRecorder.makeCoaddInputs()) #remember to set metadata if you want any hope of running detection and measurement on this coadd: #self.assembleMetadata(coaddExposure, tempExpRefList, weightList) #most important thing is the psf coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter()) coaddInputs = coaddExposure.getInfo().getCoaddInputs() for tempExp, weight in zip(coaddTempExpDict[a].values(), imageScalerRes.weightList): assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) #takes numCcds as argument coaddInputs.ccds.reserve(ccdsinPatch) coaddInputs.visits.reserve(len(imageScalerRes.dataIdList)) psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs()) coaddExposure.setPsf(psf) maskedImageList = afwImage.vectorMaskedImageF() coaddMaskedImage = coaddExposure.getMaskedImage() for dataId, imageScaler, exposure in zip( imageScalerRes.dataIdList, imageScalerRes.imageScalerList, coaddTempExpDict[a].values()): print dataId, imageScaler, exposure maskedImage = exposure.getMaskedImage() imageScaler.scaleMaskedImage(maskedImage) maskedImageList.append(maskedImage) maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, imageScalerRes.weightList) coaddMaskedImage.assign(maskedImage, skyInfo.bbox) coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) # write out Coadd! mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits" mergcoadds[a] = coaddExposure coaddExposure.writeFits(mergefilename) end = datetime.datetime.now() d = end - start f.write("time for creating merged co-adds:\n ") f.write(str(d.total_seconds())) f.write("\n") start = datetime.datetime.now() config = DetectCoaddSourcesConfig() detectCoaddSources = DetectCoaddSourcesTask(config=config) for a in dfgby.indices: # Detect on Coadd exp = mergcoadds[a] detRes = detectCoaddSources.runDetection(exp, idFactory=None) end = datetime.datetime.now() d = end - start f.write("time for detecting sources:\n ") f.write(str(d.total_seconds())) f.close()
# Now we shift to the spots data # These setup the image characterization and ISR isrConfig = IsrTaskConfig() isrConfig.doBias = False isrConfig.doDark = False isrConfig.doFlat = False isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doAddDistortionModel = False isrConfig.doWrite = True isrConfig.doAssembleCcd = True isrConfig.expectWcs = False isrConfig.doLinearize = False charConfig = CharacterizeImageConfig() charConfig.installSimplePsf.fwhm = 0.05 charConfig.doMeasurePsf = False charConfig.doApCorr = False charConfig.doDeblend = False charConfig.repair.doCosmicRay = False charConfig.repair.doInterpolate = False charConfig.detection.background.binSize = 128 charConfig.detection.minPixels = 5 # Now we characterize the spot sizes if not spots_already_done: if new_spot_repo: step1 = Popen("rm -rf %s"%SPOTS_REPO_DIR, shell=True) Popen.wait(step1) step2 = Popen("mkdir -p %s/rerun/test/plots"%SPOTS_REPO_DIR, shell=True)