def _stitch_exposures_good_pixel_copy(self, dest_wcs, dest_bbox, expo_list, warper, bad_pixel_mask=None): """ Return an exposure matching the dest wcs and dest_bbox that is composed of pixels from the exposures in expo_list. Uses coadd_utils.goodPixelCopy @ dest_wcs: WCS object for the destination exposure. @ dest_bbox: Bounding box for the destination exposure. @ expo_list: List of exposures to combine to form dextination exposure. @ warper: Warper to use when warping images. @ bad_pixel_mask: mask for pixels that should not be copied. All exposures need valid WCS. """ if bad_pixel_mask is None: bad_pixel_mask = afw_image.MaskU.getPlaneBitMask(["EDGE"]) dest_expo = afw_image.ExposureF(dest_bbox, dest_wcs) for j, expo in enumerate(expo_list): warped_exposure = warper.warpExposure(destWcs=dest_expo.getWcs(), srcExposure=expo, maxBBox=dest_expo.getBBox()) src_masked_image = warped_exposure.getMaskedImage() dest_masked_image = dest_expo.getMaskedImage() coadd_utils.copyGoodPixels(dest_masked_image, src_masked_image, bad_pixel_mask) return dest_expo
def createTempExp(self, calexpRefList, skyInfo, visitId=0): """Create a tempExp from inputs We iterate over the multiple calexps in a single exposure to construct the warp ("tempExp") of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return warped exposure, or None if no pixels overlap """ inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None for calExpInd, calExpRef in enumerate(calexpRefList): self.log.info("Processing calexp %d of %d for this tempExp: id=%s" % (calExpInd+1, len(calexpRefList), calExpRef.dataId)) try: ccdId = calExpRef.get("ccdExposureId", immediate=True) except Exception: ccdId = calExpInd numGoodPix = 0 try: # We augment the dataRef here with the tract, which is harmless for loading things # like calexps that don't need the tract, and necessary for meas_mosaic outputs, # which do. calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId, tract=skyInfo.tractInfo.getId()) calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted) exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix += numGoodPix self.log.logdebug("Calexp %s has %d good pixels in this patch (%.1f%%)" % (calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea())) if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(exposure.getCalib()) coaddTempExp.setFilter(exposure.getFilter()) didSetMetadata = True except Exception, e: self.log.warn("Error processing calexp %s; skipping it: %s" % (calExpRef.dataId, e)) continue inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
def createTempExp(self, calexpRefList, skyInfo, visitId=0): """Create a tempExp from inputs We iterate over the multiple calexps in a single exposure to construct the warp ("tempExp") of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return warped exposure, or None if no pixels overlap """ inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList)) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None for calExpInd, calExpRef in enumerate(calexpRefList): self.log.info("Processing calexp %d of %d for this tempExp: id=%s" % (calExpInd+1, len(calexpRefList), calExpRef.dataId)) try: ccdId = calExpRef.get("ccdExposureId", immediate=True) except Exception: ccdId = calExpInd numGoodPix = 0 try: # We augment the dataRef here with the tract, which is harmless for loading things # like calexps that don't need the tract, and necessary for meas_mosaic outputs, # which do. calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId, tract=skyInfo.tractInfo.getId()) calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted) exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix += numGoodPix self.log.logdebug("Calexp %s has %d good pixels in this patch (%.1f%%)" % (calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea())) if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(exposure.getCalib()) coaddTempExp.setFilter(exposure.getFilter()) didSetMetadata = True except Exception, e: self.log.warn("Error processing calexp %s; skipping it: %s" % (calExpRef.dataId, e)) continue inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
def stitchExposuresGoodPixelCopy(destWcs, destBBox, expoList, warper, badPixelMask = afwImage.MaskU.getPlaneBitMask(["EDGE"])): ''' Return an exposure matching the destWcs and destBBox that is composed of pixels from the exposures in expoList. Uses coadd_utils.goodPixelCopy @ destWcs: WCS object for the destination exposure. @ destBBox: Bounding box for the destination exposure. @ expoList: List of exposures to combine to form dextination exposure. @ warper: Warper to use when warping images. @ badPixelMask: mask for pixels that should not be copied. All exposures need valid WCS. ''' destExpo = afwImage.ExposureF(destBBox, destWcs) for j, expo in enumerate(expoList): warpedExposure = warper.warpExposure( destWcs = destExpo.getWcs(), srcExposure = expo, maxBBox = destExpo.getBBox()) srcMaskedImage = warpedExposure.getMaskedImage() destMaskedImage = destExpo.getMaskedImage() coaddUtils.copyGoodPixels(destMaskedImage, srcMaskedImage, badPixelMask) return destExpo
def basicImageTest(self, srcImage, destImage): refDestImage, refNumGoodPix = referenceCopyGoodPixelsImage(destImage, srcImage) numGoodPix = coaddUtils.copyGoodPixels(destImage, srcImage) msg = "image != reference image" try: self.assertImagesNearlyEqual(destImage, refDestImage, msg=msg) except Exception: destImage.writeFits("destImage.fits") refDestImage.writeFits("refDestImage.fits") raise self.assertEqual(numGoodPix, refNumGoodPix)
def basicMaskedImageTest(self, srcImage, destImage, badMask): refDestImage, refNumGoodPix = referenceCopyGoodPixelsMaskedImage(destImage, srcImage, badMask) numGoodPix = coaddUtils.copyGoodPixels(destImage, srcImage, badMask) self.assertEqual(numGoodPix, refNumGoodPix) msg = "masked image != reference masked image" try: self.assertMaskedImagesNearlyEqual(destImage, refDestImage, msg=msg) except Exception: destImage.writeFits("destMaskedImage.fits") refDestImage.writeFits("refDestMaskedImage.fits") raise
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs): """Create a Warp from inputs We iterate over the multiple calexps in a single exposure to construct the warp (previously called a coaddTempExp) of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return a pipeBase Struct containing: - exposures: a dictionary containing the warps requested: "direct": direct warp if config.makeDirect "psfMatched": PSF-matched warp if config.makePsfMatched """ warpTypeList = self.getWarpTypeList() totGoodPix = {warpType: 0 for warpType in warpTypeList} didSetMetadata = {warpType: False for warpType in warpTypeList} coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList} inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList)) for warpType in warpTypeList} modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None if dataIdList is None: dataIdList = ccdIdList for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)): self.log.info("Processing calexp %d of %d for this Warp: id=%s", calExpInd+1, len(calExpList), dataId) try: warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, makeDirect=self.config.makeDirect, makePsfMatched=self.config.makePsfMatched) except Exception as e: self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) continue try: numGoodPix = {warpType: 0 for warpType in warpTypeList} for warpType in warpTypeList: exposure = warpedAndMatched.getDict()[warpType] if exposure is None: continue coaddTempExp = coaddTempExps[warpType] if didSetMetadata[warpType]: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() / exposure.getPhotoCalib().getInstFluxAtZeroMagnitude()) del mimg numGoodPix[warpType] = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix[warpType] += numGoodPix[warpType] self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s", dataId, numGoodPix[warpType], 100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType) if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]: coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) coaddTempExp.setFilter(exposure.getFilter()) coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo()) # PSF replaced with CoaddPsf after loop if and only if creating direct warp coaddTempExp.setPsf(exposure.getPsf()) didSetMetadata[warpType] = True # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) except Exception as e: self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) continue for warpType in warpTypeList: self.log.info("%sWarp has %d good pixels (%.1f%%)", warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea()) if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) if warpType == "direct": coaddTempExps[warpType].setPsf( CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, self.config.coaddPsf.makeControl())) else: if not self.config.doWriteEmptyWarps: # No good pixels. Exposure still empty coaddTempExps[warpType] = None result = pipeBase.Struct(exposures=coaddTempExps) return result
def main(): # try out one exposure #visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"] visits = ["0288976", "0288935"] ccds = [] exit(0) for i in range(1, 61): ccds.append(i) filterName = 'g' DATA_PATH = "/root/extra_home/lsst_data/" #spathprefix = "/home/dongfang/download/lsst_data/" spathprefix = DATA_PATH + "raw/" #calexpsloc = "/home/dongfang/download/lsst_data/calexps/" calexpsloc = DATA_PATH + "calexps/" #coaddloc = "/home/dongfang/download/lsst_data/coadds/" coaddloc = DATA_PATH + "coadds/" #mergecoaddloc = "/home/dongfang/download/lsst_data/merge/" mergecoaddloc = DATA_PATH + "merge/" # Characterize Image charImageConfig = CharacterizeImageConfig() charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig( projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] coaddTempDict = {} calibResDict = {} f = open("log.txt", 'wb') start = datetime.datetime.now() #process CCDs to create calexps. for v in visits: for ccd in ccds: visit = int(v) filename = "instcal" + v + "." + str(ccd) + ".fits" calexpfn = calexpsloc + v + "/" + filename source = spathprefix + v + "/" + filename exposure = afwImg.ExposureF(source) try: # Characterize Image charRes = charImage.characterize(exposure, exposureIdInfo=None, background=None) except: f.write("DFZ DEBUG at charRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: # Caliberate Image calibRes = calibrateTask.calibrate( charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except: f.write("DFZ DEBUG at calibRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: #write out calexps calibRes.exposure.writeFits(calexpfn) #calbresDict.append((v,ccd),calibRes) except: f.write("DFZ DEBUG at calibRes.exposure: errors in visit " + v + ", ccd " + str(ccd) + "\n") end = datetime.datetime.now() d = end - start f.write("time for creating calexps: ") f.write(str(d.total_seconds())) f.write("\n") #time for creating co-add tempexps. start = datetime.datetime.now() # map calexps to patch-ids visit = visits[0] ccdsPerPatch = [] for ccd in ccds: filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename exposure = afwImg.ExposureF(source) bbox = exposure.getBBox() wcs = exposure.getWcs() corners = bbox.getCorners() xIndexMax, yIndexMax = tract.findPatch( wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex() xIndexMin, yIndexMin = tract.findPatch( wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex() yy = range(yIndexMin, yIndexMax + 1) xx = range(xIndexMin, xIndexMax + 1) for yIdx in yy: for xIdx in xx: ccdsPerPatch.append((ccd, (xIdx, yIdx))) print len(ccdsPerPatch) #import cPickle #cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch) # import cPickle # f = open("ccdsInPatch.p",'wb') # cPickle.dump(ccdsInPatch,f) #import cPickle #ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb')) df = pd.DataFrame(ccdsPerPatch) dfgby = df.groupby(1) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) coaddTempExpDict = {} for visit in visits: for a in dfgby.indices: coaddTempExpDict[a] = {} xInd = a[0] yInd = a[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set( numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby.get_group(a)[0].ravel(): print a print b if not setInputRecorder: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) try: inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) except: f.write("DFZ DEBUG at inputRecorder\n") setInputRecorder = True numGoodPix = 0 ccd = b filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename calExp = afwImg.ExposureF(source) ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf( modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) coaddTempExpDict[a][v] = coaddTempExp coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str( xInd) + "_" + str(yInd) + ".fits" coaddTempExp.writeFits(coaddfilename) end = datetime.datetime.now() d = end - start f.write("time for creating co-add tempexps:\n ") f.write(str(d.total_seconds())) f.write("\n") #DFZ: stop here exit(0) start = datetime.datetime.now() config = AssembleCoaddConfig() assembleTask = AssembleCoaddTask(config=config) mergcoadds = {} for a in dfgby.indices: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) xInd = a[0] yInd = a[1] imageScalerRes = prepareInputs(coaddTempExpDict[a].values(), coaddTempExpDict[a].keys(), assembleTask) mask = None doClip = False if mask is None: mask = assembleTask.getBadPixelMask() statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip) statsCtrl.setNumIter(assembleTask.config.clipIter) statsCtrl.setAndMask(mask) statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) for plane, threshold in assembleTask.config.maskPropagationThresholds.items( ): bit = afwImage.MaskU.getMaskPlane(plane) statsCtrl.setMaskPropagationThreshold(bit, threshold) if doClip: statsFlags = afwMath.MEANCLIP else: statsFlags = afwMath.MEAN coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib()) coaddExposure.getInfo().setCoaddInputs( assembleTask.inputRecorder.makeCoaddInputs()) #remember to set metadata if you want any hope of running detection and measurement on this coadd: #self.assembleMetadata(coaddExposure, tempExpRefList, weightList) #most important thing is the psf coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter()) coaddInputs = coaddExposure.getInfo().getCoaddInputs() for tempExp, weight in zip(coaddTempExpDict[a].values(), imageScalerRes.weightList): assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) #takes numCcds as argument coaddInputs.ccds.reserve(ccdsinPatch) coaddInputs.visits.reserve(len(imageScalerRes.dataIdList)) psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs()) coaddExposure.setPsf(psf) maskedImageList = afwImage.vectorMaskedImageF() coaddMaskedImage = coaddExposure.getMaskedImage() for dataId, imageScaler, exposure in zip( imageScalerRes.dataIdList, imageScalerRes.imageScalerList, coaddTempExpDict[a].values()): print dataId, imageScaler, exposure maskedImage = exposure.getMaskedImage() imageScaler.scaleMaskedImage(maskedImage) maskedImageList.append(maskedImage) maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, imageScalerRes.weightList) coaddMaskedImage.assign(maskedImage, skyInfo.bbox) coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) # write out Coadd! mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits" mergcoadds[a] = coaddExposure coaddExposure.writeFits(mergefilename) end = datetime.datetime.now() d = end - start f.write("time for creating merged co-adds:\n ") f.write(str(d.total_seconds())) f.write("\n") start = datetime.datetime.now() config = DetectCoaddSourcesConfig() detectCoaddSources = DetectCoaddSourcesTask(config=config) for a in dfgby.indices: # Detect on Coadd exp = mergcoadds[a] detRes = detectCoaddSources.runDetection(exp, idFactory=None) end = datetime.datetime.now() d = end - start f.write("time for detecting sources:\n ") f.write(str(d.total_seconds())) f.close()
calexp_array, mask, variance) calExp = afwImage.ExposureF(maskedImage) calExp.setWcs(wcs) ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure #if didSetMetadata: # mimg = calExp.getMaskedImage() # mimg *= coaddTempExp.getCalib().getFluxMag0()[0] / (0.001 + calExp.getCalib().getFluxMag0()[0]) # del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: #inputRecorder.finish(coaddTempExp, totGoodPix) #if totGoodPix > 0 and didSetMetadata: # coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else # CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs))
def makePatch(patchId, visit, dfgby_raveled, ccdsNeeded, hits_skymap): import lsst.afw.image as afwImg from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask, MakeCoaddTempExpConfig import lsst.coadd.utils as coaddUtils import numpy import lsst.skymap as skymap from lsst.meas.algorithms import CoaddPsf import lsst.skymap as skymap makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) xInd = patchId[0] yInd = patchId[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImg.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set(numpy.nan, afwImg.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby_raveled: # dfgby.get_group(a)[0].ravel(): if not setInputRecorder: ccdsinPatch = len( dfgby_raveled) # len(dfgby.get_group(a)[0].ravel()) inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) setInputRecorder = True numGoodPix = 0 ccd = b if ccdsNeeded[(v, ccd)] is not None: calExp = ccdsNeeded[(v, ccd)].exposure else: continue ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels(coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) return coaddTempExp
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None): """Create a Warp from inputs We iterate over the multiple calexps in a single exposure to construct the warp (previously called a coaddTempExp) of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return a pipeBase Struct containing: - exposures: a dictionary containing the warps requested: "direct": direct warp if config.makeDirect "psfMatched": PSF-matched warp if config.makePsfMatched """ warpTypeList = self.getWarpTypeList() totGoodPix = {warpType: 0 for warpType in warpTypeList} didSetMetadata = {warpType: False for warpType in warpTypeList} coaddTempExps = { warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList } inputRecorder = { warpType: self.inputRecorder.makeCoaddTempExpRecorder( visitId, len(calExpList)) for warpType in warpTypeList } modelPsf = self.config.modelPsf.apply( ) if self.config.makePsfMatched else None if dataIdList is None: dataIdList = ccdIdList for calExpInd, (calExp, ccdId, dataId) in enumerate( zip(calExpList, ccdIdList, dataIdList)): self.log.info("Processing calexp %d of %d for this Warp: id=%s", calExpInd + 1, len(calExpList), dataId) try: warpedAndMatched = self.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, makeDirect=self.config.makeDirect, makePsfMatched=self.config.makePsfMatched) except Exception as e: self.log.warn( "WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) continue try: numGoodPix = {warpType: 0 for warpType in warpTypeList} for warpType in warpTypeList: exposure = warpedAndMatched.getDict()[warpType] if exposure is None: continue coaddTempExp = coaddTempExps[warpType] if didSetMetadata[warpType]: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getPhotoCalib( ).getInstFluxAtZeroMagnitude() / exposure. getPhotoCalib().getInstFluxAtZeroMagnitude()) del mimg numGoodPix[warpType] = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix[warpType] += numGoodPix[warpType] self.log.debug( "Calexp %s has %d good pixels in this patch (%.1f%%) for %s", dataId, numGoodPix[warpType], 100.0 * numGoodPix[warpType] / skyInfo.bbox.getArea(), warpType) if numGoodPix[warpType] > 0 and not didSetMetadata[ warpType]: coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) coaddTempExp.setFilter(exposure.getFilter()) coaddTempExp.getInfo().setVisitInfo( exposure.getInfo().getVisitInfo()) # PSF replaced with CoaddPsf after loop if and only if creating direct warp coaddTempExp.setPsf(exposure.getPsf()) didSetMetadata[warpType] = True # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) except Exception as e: self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) continue for warpType in warpTypeList: self.log.info( "%sWarp has %d good pixels (%.1f%%)", warpType, totGoodPix[warpType], 100.0 * totGoodPix[warpType] / skyInfo.bbox.getArea()) if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) if warpType == "direct": coaddTempExps[warpType].setPsf( CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, self.config.coaddPsf.makeControl())) else: if not self.config.doWriteEmptyWarps: # No good pixels. Exposure still empty coaddTempExps[warpType] = None result = pipeBase.Struct(exposures=coaddTempExps) return result
def run(self, patchRef): """Produce <coaddName>Coadd_tempExp images and (optional) <coaddName>Coadd_initPsf <coaddName>Coadd_tempExp are produced by PSF-matching (optional) and warping. If PSF-matching is used then <coaddName>Coadd_initPsf is also computed. PSF matching is to a double gaussian model with core FWHM = self.config.warpAndPsfMatch.desiredFwhm and wings of amplitude 1/10 of core and FWHM = 2.5 * core. PSF-matching is performed before warping so the code can use the PSF models associated with the calibrated science exposures (without having to warp those models). @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", plus the camera-specific filter key (e.g. "filter" or "band") @return: a pipeBase.Struct with fields: - dataRefList: a list of data references for the new <coaddName>Coadd_tempExp @warning: this task assumes that all exposures in a coaddTempExp have the same filter. @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored (assembleCoadd should determine zeropoint scaling without referring to it). """ skyInfo = self.getSkyInfo(patchRef) tractWcs = skyInfo.wcs patchBBox = skyInfo.bbox calExpRefList = self.selectExposures(patchRef=patchRef, wcs=tractWcs, bbox=patchBBox) # initialize outputs dataRefList = [] numExp = len(calExpRefList) if numExp < 1: raise pipeBase.TaskError("No exposures to coadd") self.log.info("Process %s calexp" % (numExp,)) doPsfMatch = self.config.warpAndPsfMatch.desiredFwhm is not None if not doPsfMatch: self.log.info("No PSF matching will be done (desiredFwhm is None)") tempExpName = self.config.coaddName + "Coadd_tempExp" # compute tempKeyList: a tuple of ID key names in a calExpId that identify a coaddTempExp. # You must also specify tract and patch to make a complete coaddTempExp ID. butler = patchRef.butlerSubset.butler tempExpKeySet = set(butler.getKeys(datasetType=tempExpName, level="Ccd")) - set(("patch", "tract")) tempExpKeyList = tuple(sorted(tempExpKeySet)) # compute tempExpIdDict, a dict whose: # - keys are tuples of coaddTempExp ID values in tempKeyList order # - values are a list of calExp data references for calExp that belong in this coaddTempExp tempExpIdDict = dict() for calExpRef in calExpRefList: calExpId = calExpRef.dataId if not calExpRef.datasetExists("calexp"): self.log.warn("Could not find calexp %s; skipping it" % (calExpId,)) continue tempExpIdTuple = tuple(calExpId[key] for key in tempExpKeyList) calExpSubsetRefList = tempExpIdDict.get(tempExpIdTuple) if calExpSubsetRefList: calExpSubsetRefList.append(calExpRef) else: tempExpIdDict[tempExpIdTuple] = [calExpRef] numTempExp = len(tempExpIdDict) coaddTempExp = None for tempExpInd, calExpSubsetRefList in enumerate(tempExpIdDict.itervalues()): # derive tempExpId from the first calExpId tempExpId = dict((key, calExpSubsetRefList[0].dataId[key]) for key in tempExpKeyList) tempExpId.update(patchRef.dataId) tempExpRef = calExpRef.butlerSubset.butler.dataRef( datasetType = tempExpName, dataId = tempExpId, ) if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=tempExpName): self.log.info("tempCoaddExp exists %s; skipping it"%(tempExpId)) continue self.log.info("Computing coaddTempExp %d of %d: id=%s" % (tempExpInd+1, numTempExp, tempExpId)) totGoodPix = 0 coaddTempExp = afwImage.ExposureF(patchBBox, tractWcs) edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE") coaddTempExp.getMaskedImage().set(numpy.nan, edgeMask, numpy.inf) didSetMetadata = False for calExpInd, calExpRef in enumerate(calExpSubsetRefList): self.log.info("Processing calexp %d of %d for this tempExp: id=%s" % \ (calExpInd+1, len(calExpSubsetRefList), calExpRef.dataId)) try: exposure = self.warpAndPsfMatch.getCalExp(calExpRef, getPsf=doPsfMatch, bgSubtracted=self.config.bgSubtracted) exposure = self.warpAndPsfMatch.run(exposure, wcs=tractWcs, maxBBox=patchBBox).exposure numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self._badPixelMask) totGoodPix += numGoodPix if numGoodPix == 0: self.log.warn("Calexp %s has no good pixels in this patch" % (calExpRef.dataId,)) else: self.log.info("Calexp %s has %s good pixels in this patch" % \ (calExpRef.dataId, numGoodPix)) if not didSetMetadata: coaddTempExp.setCalib(exposure.getCalib()) coaddTempExp.setFilter(exposure.getFilter()) didSetMetadata = True except Exception, e: self.log.warn("Error processing calexp %s; skipping it: %s" % \ (calExpRef.dataId, e)) continue if (totGoodPix == 0) or not didSetMetadata: # testing didSetMetadata is not needed but safer self.log.warn("Could not compute coaddTempExp %s: no good pixels" % (tempExpRef.dataId,)) continue self.log.info("coaddTempExp %s has %s good pixels" % (tempExpRef.dataId, totGoodPix)) if self.config.doWrite and coaddTempExp is not None: self.log.info("Persisting %s %s" % (tempExpName, tempExpRef.dataId)) tempExpRef.put(coaddTempExp, tempExpName) if self.config.warpAndPsfMatch.desiredFwhm is not None: psfName = self.config.coaddName + "Coadd_initPsf" self.log.info("Persisting %s %s" % (psfName, tempExpRef.dataId)) wcs = coaddTempExp.getWcs() fwhmPixels = self.config.warpAndPsfMatch.desiredFwhm / wcs.pixelScale().asArcseconds() kernelSize = int(round(fwhmPixels * self.config.coaddKernelSizeFactor)) kernelDim = afwGeom.Point2I(kernelSize, kernelSize) coaddPsf = self.makeModelPsf(fwhmPixels=fwhmPixels, kernelDim=kernelDim) patchRef.put(coaddPsf, psfName) if coaddTempExp: dataRefList.append(tempExpRef) else: self.log.warn("This %s temp coadd exposure could not be created"%(tempExpRef.dataId,))