def finish(self, coaddTempExp, nGoodPix=None): """Finish creating the CoaddInputs for a CoaddTempExp. @param[in,out] coaddTempExp Exposure object from which to obtain the PSF, WCS, and bounding box for the entry in the 'visits' table. On return, the completed CoaddInputs object will be attached to it. @param[in] nGoodPix Total number of good pixels in the CoaddTempExp; ignored unless saveVisitGoodPix is true. """ self._setExposureInfoInRecord(exposure=coaddTempExp, record=self.visitRecord) if self.task.config.saveVisitGoodPix: self.visitRecord.setI(self.task.visitGoodPixKey, nGoodPix) coaddTempExp.getInfo().setCoaddInputs(self.coaddInputs) wcs = coaddTempExp.getWcs() if False: # This causes a test failure, pending fix in issue HSC-802 coaddTempExp.setPsf(CoaddPsf(self.coaddInputs.ccds, wcs)) apCorrMap = makeCoaddApCorrMap(self.coaddInputs.ccds, coaddTempExp.getBBox(afwImage.PARENT), wcs) coaddTempExp.getInfo().setApCorrMap(apCorrMap)
def run(self, exposure, sensorRef, templateIdList=None): """Retrieve and mosaic a template coadd that overlaps the exposure where the template spans multiple tracts. The resulting template image will be an average of all the input templates from the separate tracts. The PSF on the template is created by combining the CoaddPsf on each template image into a meta-CoaddPsf. Parameters ---------- exposure: `lsst.afw.image.Exposure` an exposure for which to generate an overlapping template sensorRef : TYPE a Butler data reference that can be used to obtain coadd data templateIdList : TYPE, optional list of data ids (unused) Returns ------- result : `struct` return a pipeBase.Struct: - ``exposure`` : a template coadd exposure assembled out of patches - ``sources`` : None for this subtask """ # Table for CoaddPSF tractsSchema = afwTable.ExposureTable.makeMinimalSchema() tractKey = tractsSchema.addField('tract', type=np.int32, doc='Which tract') patchKey = tractsSchema.addField('patch', type=np.int32, doc='Which patch') weightKey = tractsSchema.addField( 'weight', type=float, doc='Weight for each tract, should be 1') tractsCatalog = afwTable.ExposureCatalog(tractsSchema) skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") expWcs = exposure.getWcs() expBoxD = geom.Box2D(exposure.getBBox()) expBoxD.grow(self.config.templateBorderSize) ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) centralTractInfo = skyMap.findTract(ctrSkyPos) if not centralTractInfo: raise RuntimeError("No suitable tract found for central point") self.log.info("Central skyMap tract %s" % (centralTractInfo.getId(), )) skyCorners = [ expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners() ] tractPatchList = skyMap.findTractPatchList(skyCorners) if not tractPatchList: raise RuntimeError("No suitable tract found") self.log.info("All overlapping skyMap tracts %s" % ([a[0].getId() for a in tractPatchList])) # Move central tract to front of the list and use as the reference tracts = [tract[0].getId() for tract in tractPatchList] centralIndex = tracts.index(centralTractInfo.getId()) tracts.insert(0, tracts.pop(centralIndex)) tractPatchList.insert(0, tractPatchList.pop(centralIndex)) coaddPsf = None coaddFilter = None nPatchesFound = 0 maskedImageList = [] weightList = [] for itract, tract in enumerate(tracts): tractInfo = tractPatchList[itract][0] coaddWcs = tractInfo.getWcs() coaddBBox = geom.Box2D() for skyPos in skyCorners: coaddBBox.include(coaddWcs.skyToPixel(skyPos)) coaddBBox = geom.Box2I(coaddBBox) if itract == 0: # Define final wcs and bounding box from the reference tract finalWcs = coaddWcs finalBBox = coaddBBox patchList = tractPatchList[itract][1] for patchInfo in patchList: self.log.info('Adding patch %s from tract %s' % (patchInfo.getIndex(), tract)) # Local patch information patchSubBBox = geom.Box2I(patchInfo.getInnerBBox()) patchSubBBox.clip(coaddBBox) patchInt = int( f"{patchInfo.getIndex()[0]}{patchInfo.getIndex()[1]}") innerBBox = geom.Box2I(tractInfo._minimumBoundingBox(finalWcs)) if itract == 0: # clip to image and tract boundaries patchSubBBox.clip(finalBBox) patchSubBBox.clip(innerBBox) if patchSubBBox.getArea() == 0: self.log.debug("No ovlerap for patch %s" % patchInfo) continue patchArgDict = dict( datasetType="deepCoadd_sub", bbox=patchSubBBox, tract=tractInfo.getId(), patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), filter=exposure.getFilter().getName()) coaddPatch = sensorRef.get(**patchArgDict) if coaddFilter is None: coaddFilter = coaddPatch.getFilter() # create full image from final bounding box exp = afwImage.ExposureF(finalBBox, finalWcs) exp.maskedImage.set( np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) exp.maskedImage.assign(coaddPatch.maskedImage, patchSubBBox) maskedImageList.append(exp.maskedImage) weightList.append(1) record = tractsCatalog.addNew() record.setPsf(coaddPatch.getPsf()) record.setWcs(coaddPatch.getWcs()) record.setPhotoCalib(coaddPatch.getPhotoCalib()) record.setBBox(patchSubBBox) record.set(tractKey, tract) record.set(patchKey, patchInt) record.set(weightKey, 1.) nPatchesFound += 1 else: # compute the exposure bounding box in a tract that is not the reference tract localBox = geom.Box2I() for skyPos in skyCorners: localBox.include( geom.Point2I( tractInfo.getWcs().skyToPixel(skyPos))) # clip to patch bounding box localBox.clip(patchSubBBox) # grow border to deal with warping at edges localBox.grow(self.config.templateBorderSize) # clip to tract inner bounding box localInnerBBox = geom.Box2I( tractInfo._minimumBoundingBox(tractInfo.getWcs())) localBox.clip(localInnerBBox) patchArgDict = dict( datasetType="deepCoadd_sub", bbox=localBox, tract=tractInfo.getId(), patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), filter=exposure.getFilter().getName()) coaddPatch = sensorRef.get(**patchArgDict) # warp to reference tract wcs xyTransform = afwGeom.makeWcsPairTransform( coaddPatch.getWcs(), finalWcs) psfWarped = WarpedPsf(coaddPatch.getPsf(), xyTransform) warped = self.warper.warpExposure(finalWcs, coaddPatch, maxBBox=finalBBox) # check if warpped image is viable if warped.getBBox().getArea() == 0: self.log.info( "No ovlerap for warped patch %s. Skipping" % patchInfo) continue warped.setPsf(psfWarped) exp = afwImage.ExposureF(finalBBox, finalWcs) exp.maskedImage.set( np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) exp.maskedImage.assign(warped.maskedImage, warped.getBBox()) maskedImageList.append(exp.maskedImage) weightList.append(1) record = tractsCatalog.addNew() record.setPsf(psfWarped) record.setWcs(finalWcs) record.setPhotoCalib(coaddPatch.getPhotoCalib()) record.setBBox(warped.getBBox()) record.set(tractKey, tract) record.set(patchKey, patchInt) record.set(weightKey, 1.) nPatchesFound += 1 if nPatchesFound == 0: raise RuntimeError("No patches found!") # Combine images from individual patches together # Do not mask any values statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic) maskMap = [] statsCtrl = afwMath.StatisticsControl() statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) coaddExposure = afwImage.ExposureF(finalBBox, finalWcs) coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) xy0 = coaddExposure.getXY0() coaddExposure.maskedImage = afwMath.statisticsStack( maskedImageList, statsFlags, statsCtrl, weightList, 0, maskMap) coaddExposure.maskedImage.setXY0(xy0) coaddPsf = CoaddPsf(tractsCatalog, finalWcs, self.config.coaddPsf.makeControl()) if coaddPsf is None: raise RuntimeError("No coadd Psf found!") coaddExposure.setPsf(coaddPsf) coaddExposure.setFilter(coaddFilter) return pipeBase.Struct(exposure=coaddExposure, sources=None)
def createTempExp(self, calexpRefList, skyInfo, visitId=0): """Create a tempExp from inputs We iterate over the multiple calexps in a single exposure to construct the warp ("tempExp") of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return warped exposure, or None if no pixels overlap """ inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList)) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None for calExpInd, calExpRef in enumerate(calexpRefList): self.log.info("Processing calexp %d of %d for this tempExp: id=%s", calExpInd+1, len(calexpRefList), calExpRef.dataId) try: ccdId = calExpRef.get("ccdExposureId", immediate=True) except Exception: ccdId = calExpInd numGoodPix = 0 try: # We augment the dataRef here with the tract, which is harmless for loading things # like calexps that don't need the tract, and necessary for meas_mosaic outputs, # which do. calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId, tract=skyInfo.tractInfo.getId()) calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted) exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix += numGoodPix self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%)", calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea()) if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(exposure.getCalib()) coaddTempExp.setFilter(exposure.getFilter()) # PSF replaced with CoaddPsf after loop if and only if creating direct warp coaddTempExp.setPsf(exposure.getPsf()) didSetMetadata = True except Exception as e: self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e) continue inputRecorder.addCalExp(calExp, ccdId, numGoodPix) inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata and not self.config.doPsfMatch: coaddTempExp.setPsf(CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs)) self.log.info("coaddTempExp has %d good pixels (%.1f%%)", totGoodPix, 100.0*totGoodPix/skyInfo.bbox.getArea()) return coaddTempExp if totGoodPix > 0 and didSetMetadata else None
def main(): # try out one exposure #visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"] visits = ["0288976", "0288935"] ccds = [] exit(0) for i in range(1, 61): ccds.append(i) filterName = 'g' DATA_PATH = "/root/extra_home/lsst_data/" #spathprefix = "/home/dongfang/download/lsst_data/" spathprefix = DATA_PATH + "raw/" #calexpsloc = "/home/dongfang/download/lsst_data/calexps/" calexpsloc = DATA_PATH + "calexps/" #coaddloc = "/home/dongfang/download/lsst_data/coadds/" coaddloc = DATA_PATH + "coadds/" #mergecoaddloc = "/home/dongfang/download/lsst_data/merge/" mergecoaddloc = DATA_PATH + "merge/" # Characterize Image charImageConfig = CharacterizeImageConfig() charImage = CharacterizeImageTask() calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False) calibrateTask = CalibrateTask(config=calibrateConfig) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig( projection='STG', decList=[-4.9325280994132905], patchInnerDimensions=[2000, 2000], radiusList=[4.488775723429071], pixelScale=0.333, rotation=0.0, patchBorder=100, raList=[154.10660740464786], tractOverlap=0.0) hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig) tract = hits_skymap[0] coaddTempDict = {} calibResDict = {} f = open("log.txt", 'wb') start = datetime.datetime.now() #process CCDs to create calexps. for v in visits: for ccd in ccds: visit = int(v) filename = "instcal" + v + "." + str(ccd) + ".fits" calexpfn = calexpsloc + v + "/" + filename source = spathprefix + v + "/" + filename exposure = afwImg.ExposureF(source) try: # Characterize Image charRes = charImage.characterize(exposure, exposureIdInfo=None, background=None) except: f.write("DFZ DEBUG at charRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: # Caliberate Image calibRes = calibrateTask.calibrate( charRes.exposure, exposureIdInfo=None, background=charRes.background, icSourceCat=None) except: f.write("DFZ DEBUG at calibRes: errors in visit " + v + ", ccd " + str(ccd) + "\n") try: #write out calexps calibRes.exposure.writeFits(calexpfn) #calbresDict.append((v,ccd),calibRes) except: f.write("DFZ DEBUG at calibRes.exposure: errors in visit " + v + ", ccd " + str(ccd) + "\n") end = datetime.datetime.now() d = end - start f.write("time for creating calexps: ") f.write(str(d.total_seconds())) f.write("\n") #time for creating co-add tempexps. start = datetime.datetime.now() # map calexps to patch-ids visit = visits[0] ccdsPerPatch = [] for ccd in ccds: filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename exposure = afwImg.ExposureF(source) bbox = exposure.getBBox() wcs = exposure.getWcs() corners = bbox.getCorners() xIndexMax, yIndexMax = tract.findPatch( wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex() xIndexMin, yIndexMin = tract.findPatch( wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex() yy = range(yIndexMin, yIndexMax + 1) xx = range(xIndexMin, xIndexMax + 1) for yIdx in yy: for xIdx in xx: ccdsPerPatch.append((ccd, (xIdx, yIdx))) print len(ccdsPerPatch) #import cPickle #cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch) # import cPickle # f = open("ccdsInPatch.p",'wb') # cPickle.dump(ccdsInPatch,f) #import cPickle #ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb')) df = pd.DataFrame(ccdsPerPatch) dfgby = df.groupby(1) makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) coaddTempExpDict = {} for visit in visits: for a in dfgby.indices: coaddTempExpDict[a] = {} xInd = a[0] yInd = a[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set( numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby.get_group(a)[0].ravel(): print a print b if not setInputRecorder: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) try: inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) except: f.write("DFZ DEBUG at inputRecorder\n") setInputRecorder = True numGoodPix = 0 ccd = b filename = "instcal" + visit + "." + str(ccd) + ".fits" source = calexpsloc + visit + "/" + filename calExp = afwImg.ExposureF(source) ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf( modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) coaddTempExpDict[a][v] = coaddTempExp coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str( xInd) + "_" + str(yInd) + ".fits" coaddTempExp.writeFits(coaddfilename) end = datetime.datetime.now() d = end - start f.write("time for creating co-add tempexps:\n ") f.write(str(d.total_seconds())) f.write("\n") #DFZ: stop here exit(0) start = datetime.datetime.now() config = AssembleCoaddConfig() assembleTask = AssembleCoaddTask(config=config) mergcoadds = {} for a in dfgby.indices: ccdsinPatch = len(dfgby.get_group(a)[0].ravel()) xInd = a[0] yInd = a[1] imageScalerRes = prepareInputs(coaddTempExpDict[a].values(), coaddTempExpDict[a].keys(), assembleTask) mask = None doClip = False if mask is None: mask = assembleTask.getBadPixelMask() statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip) statsCtrl.setNumIter(assembleTask.config.clipIter) statsCtrl.setAndMask(mask) statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) for plane, threshold in assembleTask.config.maskPropagationThresholds.items( ): bit = afwImage.MaskU.getMaskPlane(plane) statsCtrl.setMaskPropagationThreshold(bit, threshold) if doClip: statsFlags = afwMath.MEANCLIP else: statsFlags = afwMath.MEAN coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib()) coaddExposure.getInfo().setCoaddInputs( assembleTask.inputRecorder.makeCoaddInputs()) #remember to set metadata if you want any hope of running detection and measurement on this coadd: #self.assembleMetadata(coaddExposure, tempExpRefList, weightList) #most important thing is the psf coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter()) coaddInputs = coaddExposure.getInfo().getCoaddInputs() for tempExp, weight in zip(coaddTempExpDict[a].values(), imageScalerRes.weightList): assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight) #takes numCcds as argument coaddInputs.ccds.reserve(ccdsinPatch) coaddInputs.visits.reserve(len(imageScalerRes.dataIdList)) psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs()) coaddExposure.setPsf(psf) maskedImageList = afwImage.vectorMaskedImageF() coaddMaskedImage = coaddExposure.getMaskedImage() for dataId, imageScaler, exposure in zip( imageScalerRes.dataIdList, imageScalerRes.imageScalerList, coaddTempExpDict[a].values()): print dataId, imageScaler, exposure maskedImage = exposure.getMaskedImage() imageScaler.scaleMaskedImage(maskedImage) maskedImageList.append(maskedImage) maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, imageScalerRes.weightList) coaddMaskedImage.assign(maskedImage, skyInfo.bbox) coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance()) # write out Coadd! mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits" mergcoadds[a] = coaddExposure coaddExposure.writeFits(mergefilename) end = datetime.datetime.now() d = end - start f.write("time for creating merged co-adds:\n ") f.write(str(d.total_seconds())) f.write("\n") start = datetime.datetime.now() config = DetectCoaddSourcesConfig() detectCoaddSources = DetectCoaddSourcesTask(config=config) for a in dfgby.indices: # Detect on Coadd exp = mergcoadds[a] detRes = detectCoaddSources.runDetection(exp, idFactory=None) end = datetime.datetime.now() d = end - start f.write("time for detecting sources:\n ") f.write(str(d.total_seconds())) f.close()
def makePatch(patchId, visit, dfgby_raveled, ccdsNeeded, hits_skymap): import lsst.afw.image as afwImg from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask, MakeCoaddTempExpConfig import lsst.coadd.utils as coaddUtils import numpy import lsst.skymap as skymap from lsst.meas.algorithms import CoaddPsf import lsst.skymap as skymap makeCTEConfig = MakeCoaddTempExpConfig() makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig) xInd = patchId[0] yInd = patchId[1] skyInfo = getSkyInfo(hits_skymap, xInd, yInd) v = int(visit) coaddTempExp = afwImg.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set(numpy.nan, afwImg.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = makeCTEConfig.modelPsf.apply( ) if makeCTEConfig.doPsfMatch else None setInputRecorder = False for b in dfgby_raveled: # dfgby.get_group(a)[0].ravel(): if not setInputRecorder: ccdsinPatch = len( dfgby_raveled) # len(dfgby.get_group(a)[0].ravel()) inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder( v, ccdsinPatch) setInputRecorder = True numGoodPix = 0 ccd = b if ccdsNeeded[(v, ccd)] is not None: calExp = ccdsNeeded[(v, ccd)].exposure else: continue ccdId = calExp.getId() warpedCcdExp = makeCTE.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = calExp.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / calExp.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels(coaddTempExp.getMaskedImage(), warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask()) totGoodPix += numGoodPix if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(warpedCcdExp.getCalib()) coaddTempExp.setFilter(warpedCcdExp.getFilter()) didSetMetadata = True inputRecorder.addCalExp(calExp, ccdId, numGoodPix) ##### End loop over ccds here: inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf( inputRecorder.coaddInputs.ccds, skyInfo.wcs)) return coaddTempExp
def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs): """Warp coadds from multiple tracts to form a template for image diff. Where the tracts overlap, the resulting template image is averaged. The PSF on the template is created by combining the CoaddPsf on each template image into a meta-CoaddPsf. Parameters ---------- coaddExposures : `list` of `lsst.afw.image.Exposure` Coadds to be mosaicked bbox : `lsst.geom.Box2I` Template Bounding box of the detector geometry onto which to resample the coaddExposures wcs : `lsst.afw.geom.SkyWcs` Template WCS onto which to resample the coaddExposures dataIds : `list` of `lsst.daf.butler.DataCoordinate` Record of the tract and patch of each coaddExposure. **kwargs Any additional keyword parameters. Returns ------- result : `lsst.pipe.base.Struct` containing - ``outputExposure`` : a template coadd exposure assembled out of patches """ # Table for CoaddPSF tractsSchema = afwTable.ExposureTable.makeMinimalSchema() tractKey = tractsSchema.addField('tract', type=np.int32, doc='Which tract') patchKey = tractsSchema.addField('patch', type=np.int32, doc='Which patch') weightKey = tractsSchema.addField( 'weight', type=float, doc='Weight for each tract, should be 1') tractsCatalog = afwTable.ExposureCatalog(tractsSchema) finalWcs = wcs bbox.grow(self.config.templateBorderSize) finalBBox = bbox nPatchesFound = 0 maskedImageList = [] weightList = [] for coaddExposure, dataId in zip(coaddExposures, dataIds): # warp to detector WCS warped = self.warper.warpExposure(finalWcs, coaddExposure, maxBBox=finalBBox) # Check if warped image is viable if not np.any(np.isfinite(warped.image.array)): self.log.info("No overlap for warped %s. Skipping" % dataId) continue exp = afwImage.ExposureF(finalBBox, finalWcs) exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) exp.maskedImage.assign(warped.maskedImage, warped.getBBox()) maskedImageList.append(exp.maskedImage) weightList.append(1) record = tractsCatalog.addNew() record.setPsf(coaddExposure.getPsf()) record.setWcs(coaddExposure.getWcs()) record.setPhotoCalib(coaddExposure.getPhotoCalib()) record.setBBox(coaddExposure.getBBox()) record.setValidPolygon( afwGeom.Polygon( geom.Box2D(coaddExposure.getBBox()).getCorners())) record.set(tractKey, dataId['tract']) record.set(patchKey, dataId['patch']) record.set(weightKey, 1.) nPatchesFound += 1 if nPatchesFound == 0: raise pipeBase.NoWorkFound("No patches found to overlap detector") # Combine images from individual patches together statsFlags = afwMath.stringToStatisticsProperty('MEAN') statsCtrl = afwMath.StatisticsControl() statsCtrl.setNanSafe(True) statsCtrl.setWeighted(True) statsCtrl.setCalcErrorFromInputVariance(True) templateExposure = afwImage.ExposureF(finalBBox, finalWcs) templateExposure.maskedImage.set( np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) xy0 = templateExposure.getXY0() # Do not mask any values templateExposure.maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList, clipped=0, maskMap=[]) templateExposure.maskedImage.setXY0(xy0) # CoaddPsf centroid not only must overlap image, but must overlap the part of # image with data. Use centroid of region with data boolmask = templateExposure.mask.array & templateExposure.mask.getPlaneBitMask( 'NO_DATA') == 0 maskx = afwImage.makeMaskFromArray(boolmask.astype(afwImage.MaskPixel)) centerCoord = afwGeom.SpanSet.fromMask(maskx, 1).computeCentroid() ctrl = self.config.coaddPsf.makeControl() coaddPsf = CoaddPsf(tractsCatalog, finalWcs, centerCoord, ctrl.warpingKernelName, ctrl.cacheSize) if coaddPsf is None: raise RuntimeError("CoaddPsf could not be constructed") templateExposure.setPsf(coaddPsf) templateExposure.setFilter(coaddExposure.getFilter()) templateExposure.setPhotoCalib(coaddExposure.getPhotoCalib()) return pipeBase.Struct(outputExposure=templateExposure)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None): """Create a Warp from inputs We iterate over the multiple calexps in a single exposure to construct the warp (previously called a coaddTempExp) of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return a pipeBase Struct containing: - exposures: a dictionary containing the warps requested: "direct": direct warp if config.makeDirect "psfMatched": PSF-matched warp if config.makePsfMatched """ warpTypeList = self.getWarpTypeList() totGoodPix = {warpType: 0 for warpType in warpTypeList} didSetMetadata = {warpType: False for warpType in warpTypeList} coaddTempExps = { warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList } inputRecorder = { warpType: self.inputRecorder.makeCoaddTempExpRecorder( visitId, len(calExpList)) for warpType in warpTypeList } modelPsf = self.config.modelPsf.apply( ) if self.config.makePsfMatched else None if dataIdList is None: dataIdList = ccdIdList for calExpInd, (calExp, ccdId, dataId) in enumerate( zip(calExpList, ccdIdList, dataIdList)): self.log.info("Processing calexp %d of %d for this Warp: id=%s", calExpInd + 1, len(calExpList), dataId) try: warpedAndMatched = self.warpAndPsfMatch.run( calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox, makeDirect=self.config.makeDirect, makePsfMatched=self.config.makePsfMatched) except Exception as e: self.log.warn( "WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e) continue try: numGoodPix = {warpType: 0 for warpType in warpTypeList} for warpType in warpTypeList: exposure = warpedAndMatched.getDict()[warpType] if exposure is None: continue coaddTempExp = coaddTempExps[warpType] if didSetMetadata[warpType]: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getPhotoCalib( ).getInstFluxAtZeroMagnitude() / exposure. getPhotoCalib().getInstFluxAtZeroMagnitude()) del mimg numGoodPix[warpType] = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix[warpType] += numGoodPix[warpType] self.log.debug( "Calexp %s has %d good pixels in this patch (%.1f%%) for %s", dataId, numGoodPix[warpType], 100.0 * numGoodPix[warpType] / skyInfo.bbox.getArea(), warpType) if numGoodPix[warpType] > 0 and not didSetMetadata[ warpType]: coaddTempExp.setPhotoCalib(exposure.getPhotoCalib()) coaddTempExp.setFilter(exposure.getFilter()) coaddTempExp.getInfo().setVisitInfo( exposure.getInfo().getVisitInfo()) # PSF replaced with CoaddPsf after loop if and only if creating direct warp coaddTempExp.setPsf(exposure.getPsf()) didSetMetadata[warpType] = True # Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType]) except Exception as e: self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e) continue for warpType in warpTypeList: self.log.info( "%sWarp has %d good pixels (%.1f%%)", warpType, totGoodPix[warpType], 100.0 * totGoodPix[warpType] / skyInfo.bbox.getArea()) if totGoodPix[warpType] > 0 and didSetMetadata[warpType]: inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType]) if warpType == "direct": coaddTempExps[warpType].setPsf( CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs, self.config.coaddPsf.makeControl())) else: if not self.config.doWriteEmptyWarps: # No good pixels. Exposure still empty coaddTempExps[warpType] = None result = pipeBase.Struct(exposures=coaddTempExps) return result
class MakeCoaddTempExpTask(CoaddBaseTask): """Task to produce <coaddName>Coadd_tempExp images """ ConfigClass = MakeCoaddTempExpConfig _DefaultName = "makeCoaddTempExp" def __init__(self, *args, **kwargs): CoaddBaseTask.__init__(self, *args, **kwargs) self.makeSubtask("warpAndPsfMatch") @pipeBase.timeMethod def run(self, patchRef, selectDataList=[]): """Produce <coaddName>Coadd_tempExp images <coaddName>Coadd_tempExp are produced by PSF-matching (optional) and warping. @param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch", plus the camera-specific filter key (e.g. "filter" or "band") @return: dataRefList: a list of data references for the new <coaddName>Coadd_tempExp @warning: this task assumes that all exposures in a coaddTempExp have the same filter. @warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored (assembleCoadd should determine zeropoint scaling without referring to it). """ skyInfo = self.getSkyInfo(patchRef) calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList) if len(calExpRefList) == 0: self.log.warn("No exposures to coadd for patch %s" % patchRef.dataId) return None self.log.info("Selected %d calexps for patch %s" % (len(calExpRefList), patchRef.dataId)) calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")] self.log.info("Processing %d existing calexps for patch %s" % (len(calExpRefList), patchRef.dataId)) groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(), self.getTempExpDatasetName()) self.log.info("Processing %d tempExps for patch %s" % (len(groupData.groups), patchRef.dataId)) dataRefList = [] for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.iteritems()): tempExpRef = getGroupDataRef(patchRef.getButler(), self.getTempExpDatasetName(), tempExpTuple, groupData.keys) if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=self.getTempExpDatasetName()): self.log.info("tempCoaddExp %s exists; skipping" % (tempExpRef.dataId,)) dataRefList.append(tempExpRef) continue self.log.info("Processing tempExp %d/%d: id=%s" % (i, len(groupData.groups), tempExpRef.dataId)) # TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776). # For now, we try to get a long integer "visit" key, and if we can't, we just use the index # of the visit in the list. try: visitId = long(tempExpRef.dataId["visit"]) except (KeyError, ValueError): visitId = i exp = self.createTempExp(calexpRefList, skyInfo, visitId) if exp is not None: dataRefList.append(tempExpRef) if self.config.doWrite: self.writeCoaddOutput(tempExpRef, exp, "tempExp") else: self.log.warn("tempExp %s could not be created" % (tempExpRef.dataId,)) return dataRefList def createTempExp(self, calexpRefList, skyInfo, visitId=0): """Create a tempExp from inputs We iterate over the multiple calexps in a single exposure to construct the warp ("tempExp") of that exposure to the supplied tract/patch. Pixels that receive no pixels are set to NAN; this is not correct (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. @param calexpRefList: List of data references for calexps that (may) overlap the patch of interest @param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric information about the patch @param visitId: integer identifier for visit, for the table that will produce the CoaddPsf @return warped exposure, or None if no pixels overlap """ inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList)) coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs) coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf) totGoodPix = 0 didSetMetadata = False modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None for calExpInd, calExpRef in enumerate(calexpRefList): self.log.info("Processing calexp %d of %d for this tempExp: id=%s" % (calExpInd+1, len(calexpRefList), calExpRef.dataId)) try: ccdId = calExpRef.get("ccdExposureId", immediate=True) except Exception: ccdId = calExpInd numGoodPix = 0 try: # We augment the dataRef here with the tract, which is harmless for loading things # like calexps that don't need the tract, and necessary for meas_mosaic outputs, # which do. calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId, tract=skyInfo.tractInfo.getId()) calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted) exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs, maxBBox=skyInfo.bbox).exposure if didSetMetadata: mimg = exposure.getMaskedImage() mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0]) del mimg numGoodPix = coaddUtils.copyGoodPixels( coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask()) totGoodPix += numGoodPix self.log.logdebug("Calexp %s has %d good pixels in this patch (%.1f%%)" % (calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea())) if numGoodPix > 0 and not didSetMetadata: coaddTempExp.setCalib(exposure.getCalib()) coaddTempExp.setFilter(exposure.getFilter()) didSetMetadata = True except Exception, e: self.log.warn("Error processing calexp %s; skipping it: %s" % (calExpRef.dataId, e)) continue inputRecorder.addCalExp(calExp, ccdId, numGoodPix) inputRecorder.finish(coaddTempExp, totGoodPix) if totGoodPix > 0 and didSetMetadata: coaddTempExp.setPsf(modelPsf if self.config.doPsfMatch else CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs)) self.log.info("coaddTempExp has %d good pixels (%.1f%%)" % (totGoodPix, 100.0*totGoodPix/skyInfo.bbox.getArea())) return coaddTempExp if totGoodPix > 0 and didSetMetadata else None