def _stitch_exposures_good_pixel_copy(self,
dest_wcs,
dest_bbox,
expo_list,
warper,
bad_pixel_mask=None):
""" Return an exposure matching the dest wcs and dest_bbox that is
composed of
pixels from the exposures in expo_list. Uses coadd_utils.goodPixelCopy
@ dest_wcs: WCS object for the destination exposure.
@ dest_bbox: Bounding box for the destination exposure.
@ expo_list: List of exposures to combine to form dextination exposure.
@ warper: Warper to use when warping images.
@ bad_pixel_mask: mask for pixels that should not be copied.
All exposures need valid WCS.
"""
if bad_pixel_mask is None:
bad_pixel_mask = afw_image.MaskU.getPlaneBitMask(["EDGE"])
dest_expo = afw_image.ExposureF(dest_bbox, dest_wcs)
for j, expo in enumerate(expo_list):
warped_exposure = warper.warpExposure(destWcs=dest_expo.getWcs(),
srcExposure=expo,
maxBBox=dest_expo.getBBox())
src_masked_image = warped_exposure.getMaskedImage()
dest_masked_image = dest_expo.getMaskedImage()
coadd_utils.copyGoodPixels(dest_masked_image, src_masked_image,
bad_pixel_mask)
return dest_expo
def createTempExp(self, calexpRefList, skyInfo, visitId=0):
"""Create a tempExp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp ("tempExp") of that exposure to the supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return warped exposure, or None if no pixels overlap
"""
inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId)
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None
for calExpInd, calExpRef in enumerate(calexpRefList):
self.log.info("Processing calexp %d of %d for this tempExp: id=%s" %
(calExpInd+1, len(calexpRefList), calExpRef.dataId))
try:
ccdId = calExpRef.get("ccdExposureId", immediate=True)
except Exception:
ccdId = calExpInd
numGoodPix = 0
try:
# We augment the dataRef here with the tract, which is harmless for loading things
# like calexps that don't need the tract, and necessary for meas_mosaic outputs,
# which do.
calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
tract=skyInfo.tractInfo.getId())
calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix += numGoodPix
self.log.logdebug("Calexp %s has %d good pixels in this patch (%.1f%%)" %
(calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea()))
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(exposure.getCalib())
coaddTempExp.setFilter(exposure.getFilter())
didSetMetadata = True
except Exception, e:
self.log.warn("Error processing calexp %s; skipping it: %s" % (calExpRef.dataId, e))
continue
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
def createTempExp(self, calexpRefList, skyInfo, visitId=0):
"""Create a tempExp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp ("tempExp") of that exposure to the supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return warped exposure, or None if no pixels overlap
"""
inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None
for calExpInd, calExpRef in enumerate(calexpRefList):
self.log.info("Processing calexp %d of %d for this tempExp: id=%s" %
(calExpInd+1, len(calexpRefList), calExpRef.dataId))
try:
ccdId = calExpRef.get("ccdExposureId", immediate=True)
except Exception:
ccdId = calExpInd
numGoodPix = 0
try:
# We augment the dataRef here with the tract, which is harmless for loading things
# like calexps that don't need the tract, and necessary for meas_mosaic outputs,
# which do.
calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
tract=skyInfo.tractInfo.getId())
calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix += numGoodPix
self.log.logdebug("Calexp %s has %d good pixels in this patch (%.1f%%)" %
(calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea()))
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(exposure.getCalib())
coaddTempExp.setFilter(exposure.getFilter())
didSetMetadata = True
except Exception, e:
self.log.warn("Error processing calexp %s; skipping it: %s" % (calExpRef.dataId, e))
continue
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
def stitchExposuresGoodPixelCopy(destWcs, destBBox, expoList, warper,
badPixelMask = afwImage.MaskU.getPlaneBitMask(["EDGE"])):
''' Return an exposure matching the destWcs and destBBox that is composed of
pixels from the exposures in expoList. Uses coadd_utils.goodPixelCopy
@ destWcs: WCS object for the destination exposure.
@ destBBox: Bounding box for the destination exposure.
@ expoList: List of exposures to combine to form dextination exposure.
@ warper: Warper to use when warping images.
@ badPixelMask: mask for pixels that should not be copied.
All exposures need valid WCS.
'''
destExpo = afwImage.ExposureF(destBBox, destWcs)
for j, expo in enumerate(expoList):
warpedExposure = warper.warpExposure(
destWcs = destExpo.getWcs(),
srcExposure = expo,
maxBBox = destExpo.getBBox())
srcMaskedImage = warpedExposure.getMaskedImage()
destMaskedImage = destExpo.getMaskedImage()
coaddUtils.copyGoodPixels(destMaskedImage, srcMaskedImage, badPixelMask)
return destExpo
def basicImageTest(self, srcImage, destImage):
refDestImage, refNumGoodPix = referenceCopyGoodPixelsImage(destImage, srcImage)
numGoodPix = coaddUtils.copyGoodPixels(destImage, srcImage)
msg = "image != reference image"
try:
self.assertImagesNearlyEqual(destImage, refDestImage, msg=msg)
except Exception:
destImage.writeFits("destImage.fits")
refDestImage.writeFits("refDestImage.fits")
raise
self.assertEqual(numGoodPix, refNumGoodPix)
def basicMaskedImageTest(self, srcImage, destImage, badMask):
refDestImage, refNumGoodPix = referenceCopyGoodPixelsMaskedImage(destImage, srcImage, badMask)
numGoodPix = coaddUtils.copyGoodPixels(destImage, srcImage, badMask)
self.assertEqual(numGoodPix, refNumGoodPix)
msg = "masked image != reference masked image"
try:
self.assertMaskedImagesNearlyEqual(destImage, refDestImage, msg=msg)
except Exception:
destImage.writeFits("destMaskedImage.fits")
refDestImage.writeFits("refDestMaskedImage.fits")
raise
def basicImageTest(self, srcImage, destImage):
refDestImage, refNumGoodPix = referenceCopyGoodPixelsImage(destImage, srcImage)
numGoodPix = coaddUtils.copyGoodPixels(destImage, srcImage)
msg = "image != reference image"
try:
self.assertImagesNearlyEqual(destImage, refDestImage, msg=msg)
except Exception:
destImage.writeFits("destImage.fits")
refDestImage.writeFits("refDestImage.fits")
raise
self.assertEqual(numGoodPix, refNumGoodPix)
def basicMaskedImageTest(self, srcImage, destImage, badMask):
refDestImage, refNumGoodPix = referenceCopyGoodPixelsMaskedImage(destImage, srcImage, badMask)
numGoodPix = coaddUtils.copyGoodPixels(destImage, srcImage, badMask)
self.assertEqual(numGoodPix, refNumGoodPix)
msg = "masked image != reference masked image"
try:
self.assertMaskedImagesNearlyEqual(destImage, refDestImage, msg=msg)
except Exception:
destImage.writeFits("destMaskedImage.fits")
refDestImage.writeFits("refDestMaskedImage.fits")
raise
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwargs):
"""Create a Warp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp (previously called a coaddTempExp) of that exposure to the
supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return a pipeBase Struct containing:
- exposures: a dictionary containing the warps requested:
"direct": direct warp if config.makeDirect
"psfMatched": PSF-matched warp if config.makePsfMatched
"""
warpTypeList = self.getWarpTypeList()
totGoodPix = {warpType: 0 for warpType in warpTypeList}
didSetMetadata = {warpType: False for warpType in warpTypeList}
coaddTempExps = {warpType: self._prepareEmptyExposure(skyInfo) for warpType in warpTypeList}
inputRecorder = {warpType: self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calExpList))
for warpType in warpTypeList}
modelPsf = self.config.modelPsf.apply() if self.config.makePsfMatched else None
if dataIdList is None:
dataIdList = ccdIdList
for calExpInd, (calExp, ccdId, dataId) in enumerate(zip(calExpList, ccdIdList, dataIdList)):
self.log.info("Processing calexp %d of %d for this Warp: id=%s",
calExpInd+1, len(calExpList), dataId)
try:
warpedAndMatched = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf,
wcs=skyInfo.wcs, maxBBox=skyInfo.bbox,
makeDirect=self.config.makeDirect,
makePsfMatched=self.config.makePsfMatched)
except Exception as e:
self.log.warn("WarpAndPsfMatch failed for calexp %s; skipping it: %s", dataId, e)
continue
try:
numGoodPix = {warpType: 0 for warpType in warpTypeList}
for warpType in warpTypeList:
exposure = warpedAndMatched.getDict()[warpType]
if exposure is None:
continue
coaddTempExp = coaddTempExps[warpType]
if didSetMetadata[warpType]:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getPhotoCalib().getInstFluxAtZeroMagnitude() /
exposure.getPhotoCalib().getInstFluxAtZeroMagnitude())
del mimg
numGoodPix[warpType] = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix[warpType] += numGoodPix[warpType]
self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
dataId, numGoodPix[warpType],
100.0*numGoodPix[warpType]/skyInfo.bbox.getArea(), warpType)
if numGoodPix[warpType] > 0 and not didSetMetadata[warpType]:
coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
coaddTempExp.setFilter(exposure.getFilter())
coaddTempExp.getInfo().setVisitInfo(exposure.getInfo().getVisitInfo())
# PSF replaced with CoaddPsf after loop if and only if creating direct warp
coaddTempExp.setPsf(exposure.getPsf())
didSetMetadata[warpType] = True
# Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
inputRecorder[warpType].addCalExp(calExp, ccdId, numGoodPix[warpType])
except Exception as e:
self.log.warn("Error processing calexp %s; skipping it: %s", dataId, e)
continue
for warpType in warpTypeList:
self.log.info("%sWarp has %d good pixels (%.1f%%)",
warpType, totGoodPix[warpType], 100.0*totGoodPix[warpType]/skyInfo.bbox.getArea())
if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
inputRecorder[warpType].finish(coaddTempExps[warpType], totGoodPix[warpType])
if warpType == "direct":
coaddTempExps[warpType].setPsf(
CoaddPsf(inputRecorder[warpType].coaddInputs.ccds, skyInfo.wcs,
self.config.coaddPsf.makeControl()))
else:
if not self.config.doWriteEmptyWarps:
# No good pixels. Exposure still empty
coaddTempExps[warpType] = None
result = pipeBase.Struct(exposures=coaddTempExps)
return result
def main():
# try out one exposure
#visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"]
visits = ["0288976", "0288935"]
ccds = []
exit(0)
for i in range(1, 61):
ccds.append(i)
filterName = 'g'
DATA_PATH = "/root/extra_home/lsst_data/"
#spathprefix = "/home/dongfang/download/lsst_data/"
spathprefix = DATA_PATH + "raw/"
#calexpsloc = "/home/dongfang/download/lsst_data/calexps/"
calexpsloc = DATA_PATH + "calexps/"
#coaddloc = "/home/dongfang/download/lsst_data/coadds/"
coaddloc = DATA_PATH + "coadds/"
#mergecoaddloc = "/home/dongfang/download/lsst_data/merge/"
mergecoaddloc = DATA_PATH + "merge/"
# Characterize Image
charImageConfig = CharacterizeImageConfig()
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(
projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333,
rotation=0.0,
patchBorder=100,
raList=[154.10660740464786],
tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
coaddTempDict = {}
calibResDict = {}
f = open("log.txt", 'wb')
start = datetime.datetime.now()
#process CCDs to create calexps.
for v in visits:
for ccd in ccds:
visit = int(v)
filename = "instcal" + v + "." + str(ccd) + ".fits"
calexpfn = calexpsloc + v + "/" + filename
source = spathprefix + v + "/" + filename
exposure = afwImg.ExposureF(source)
try:
# Characterize Image
charRes = charImage.characterize(exposure,
exposureIdInfo=None,
background=None)
except:
f.write("DFZ DEBUG at charRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
# Caliberate Image
calibRes = calibrateTask.calibrate(
charRes.exposure,
exposureIdInfo=None,
background=charRes.background,
icSourceCat=None)
except:
f.write("DFZ DEBUG at calibRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
#write out calexps
calibRes.exposure.writeFits(calexpfn)
#calbresDict.append((v,ccd),calibRes)
except:
f.write("DFZ DEBUG at calibRes.exposure: errors in visit " +
v + ", ccd " + str(ccd) + "\n")
end = datetime.datetime.now()
d = end - start
f.write("time for creating calexps: ")
f.write(str(d.total_seconds()))
f.write("\n")
#time for creating co-add tempexps.
start = datetime.datetime.now()
# map calexps to patch-ids
visit = visits[0]
ccdsPerPatch = []
for ccd in ccds:
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
exposure = afwImg.ExposureF(source)
bbox = exposure.getBBox()
wcs = exposure.getWcs()
corners = bbox.getCorners()
xIndexMax, yIndexMax = tract.findPatch(
wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex()
xIndexMin, yIndexMin = tract.findPatch(
wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex()
yy = range(yIndexMin, yIndexMax + 1)
xx = range(xIndexMin, xIndexMax + 1)
for yIdx in yy:
for xIdx in xx:
ccdsPerPatch.append((ccd, (xIdx, yIdx)))
print len(ccdsPerPatch)
#import cPickle
#cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch)
# import cPickle
# f = open("ccdsInPatch.p",'wb')
# cPickle.dump(ccdsInPatch,f)
#import cPickle
#ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb'))
df = pd.DataFrame(ccdsPerPatch)
dfgby = df.groupby(1)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
coaddTempExpDict = {}
for visit in visits:
for a in dfgby.indices:
coaddTempExpDict[a] = {}
xInd = a[0]
yInd = a[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(
numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby.get_group(a)[0].ravel():
print a
print b
if not setInputRecorder:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
try:
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
except:
f.write("DFZ DEBUG at inputRecorder\n")
setInputRecorder = True
numGoodPix = 0
ccd = b
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
calExp = afwImg.ExposureF(source)
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(
modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
coaddTempExpDict[a][v] = coaddTempExp
coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str(
xInd) + "_" + str(yInd) + ".fits"
coaddTempExp.writeFits(coaddfilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating co-add tempexps:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
#DFZ: stop here
exit(0)
start = datetime.datetime.now()
config = AssembleCoaddConfig()
assembleTask = AssembleCoaddTask(config=config)
mergcoadds = {}
for a in dfgby.indices:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
xInd = a[0]
yInd = a[1]
imageScalerRes = prepareInputs(coaddTempExpDict[a].values(),
coaddTempExpDict[a].keys(),
assembleTask)
mask = None
doClip = False
if mask is None:
mask = assembleTask.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip)
statsCtrl.setNumIter(assembleTask.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
for plane, threshold in assembleTask.config.maskPropagationThresholds.items(
):
bit = afwImage.MaskU.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
if doClip:
statsFlags = afwMath.MEANCLIP
else:
statsFlags = afwMath.MEAN
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib())
coaddExposure.getInfo().setCoaddInputs(
assembleTask.inputRecorder.makeCoaddInputs())
#remember to set metadata if you want any hope of running detection and measurement on this coadd:
#self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
#most important thing is the psf
coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter())
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
for tempExp, weight in zip(coaddTempExpDict[a].values(),
imageScalerRes.weightList):
assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp,
weight)
#takes numCcds as argument
coaddInputs.ccds.reserve(ccdsinPatch)
coaddInputs.visits.reserve(len(imageScalerRes.dataIdList))
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
coaddExposure.setPsf(psf)
maskedImageList = afwImage.vectorMaskedImageF()
coaddMaskedImage = coaddExposure.getMaskedImage()
for dataId, imageScaler, exposure in zip(
imageScalerRes.dataIdList, imageScalerRes.imageScalerList,
coaddTempExpDict[a].values()):
print dataId, imageScaler, exposure
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
maskedImageList.append(maskedImage)
maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags,
statsCtrl,
imageScalerRes.weightList)
coaddMaskedImage.assign(maskedImage, skyInfo.bbox)
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(),
coaddMaskedImage.getVariance())
# write out Coadd!
mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits"
mergcoadds[a] = coaddExposure
coaddExposure.writeFits(mergefilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating merged co-adds:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
start = datetime.datetime.now()
config = DetectCoaddSourcesConfig()
detectCoaddSources = DetectCoaddSourcesTask(config=config)
for a in dfgby.indices:
# Detect on Coadd
exp = mergcoadds[a]
detRes = detectCoaddSources.runDetection(exp, idFactory=None)
end = datetime.datetime.now()
d = end - start
f.write("time for detecting sources:\n ")
f.write(str(d.total_seconds()))
f.close()
calexp_array, mask, variance)
calExp = afwImage.ExposureF(maskedImage)
calExp.setWcs(wcs)
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
#if didSetMetadata:
# mimg = calExp.getMaskedImage()
# mimg *= coaddTempExp.getCalib().getFluxMag0()[0] / (0.001 + calExp.getCalib().getFluxMag0()[0])
# del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
#inputRecorder.finish(coaddTempExp, totGoodPix)
#if totGoodPix > 0 and didSetMetadata:
# coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else
# CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs))
def makePatch(patchId, visit, dfgby_raveled, ccdsNeeded, hits_skymap):
import lsst.afw.image as afwImg
from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask, MakeCoaddTempExpConfig
import lsst.coadd.utils as coaddUtils
import numpy
import lsst.skymap as skymap
from lsst.meas.algorithms import CoaddPsf
import lsst.skymap as skymap
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
xInd = patchId[0]
yInd = patchId[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImg.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan,
afwImg.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby_raveled: # dfgby.get_group(a)[0].ravel():
if not setInputRecorder:
ccdsinPatch = len(
dfgby_raveled) # len(dfgby.get_group(a)[0].ravel())
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
setInputRecorder = True
numGoodPix = 0
ccd = b
if ccdsNeeded[(v, ccd)] is not None:
calExp = ccdsNeeded[(v, ccd)].exposure
else:
continue
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(),
makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
return coaddTempExp
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None):
"""Create a Warp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp (previously called a coaddTempExp) of that exposure to the
supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return a pipeBase Struct containing:
- exposures: a dictionary containing the warps requested:
"direct": direct warp if config.makeDirect
"psfMatched": PSF-matched warp if config.makePsfMatched
"""
warpTypeList = self.getWarpTypeList()
totGoodPix = {warpType: 0 for warpType in warpTypeList}
didSetMetadata = {warpType: False for warpType in warpTypeList}
coaddTempExps = {
warpType: self._prepareEmptyExposure(skyInfo)
for warpType in warpTypeList
}
inputRecorder = {
warpType: self.inputRecorder.makeCoaddTempExpRecorder(
visitId, len(calExpList))
for warpType in warpTypeList
}
modelPsf = self.config.modelPsf.apply(
) if self.config.makePsfMatched else None
if dataIdList is None:
dataIdList = ccdIdList
for calExpInd, (calExp, ccdId, dataId) in enumerate(
zip(calExpList, ccdIdList, dataIdList)):
self.log.info("Processing calexp %d of %d for this Warp: id=%s",
calExpInd + 1, len(calExpList), dataId)
try:
warpedAndMatched = self.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox,
makeDirect=self.config.makeDirect,
makePsfMatched=self.config.makePsfMatched)
except Exception as e:
self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s",
dataId, e)
continue
try:
numGoodPix = {warpType: 0 for warpType in warpTypeList}
for warpType in warpTypeList:
exposure = warpedAndMatched.getDict()[warpType]
if exposure is None:
continue
coaddTempExp = coaddTempExps[warpType]
if didSetMetadata[warpType]:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getPhotoCalib(
).getInstFluxAtZeroMagnitude() / exposure.
getPhotoCalib().getInstFluxAtZeroMagnitude())
del mimg
numGoodPix[warpType] = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix[warpType] += numGoodPix[warpType]
self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
dataId, numGoodPix[warpType],
100.0 * numGoodPix[warpType] / skyInfo.bbox.getArea(),
warpType)
if numGoodPix[warpType] > 0 and not didSetMetadata[
warpType]:
coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
coaddTempExp.setFilter(exposure.getFilter())
coaddTempExp.getInfo().setVisitInfo(
exposure.getInfo().getVisitInfo())
# PSF replaced with CoaddPsf after loop if and only if creating direct warp
coaddTempExp.setPsf(exposure.getPsf())
didSetMetadata[warpType] = True
# Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
inputRecorder[warpType].addCalExp(calExp, ccdId,
numGoodPix[warpType])
except Exception as e:
self.log.warn("Error processing calexp %s; skipping it: %s",
dataId, e)
continue
for warpType in warpTypeList:
self.log.info(
"%sWarp has %d good pixels (%.1f%%)", warpType,
totGoodPix[warpType],
100.0 * totGoodPix[warpType] / skyInfo.bbox.getArea())
if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
inputRecorder[warpType].finish(coaddTempExps[warpType],
totGoodPix[warpType])
if warpType == "direct":
coaddTempExps[warpType].setPsf(
CoaddPsf(inputRecorder[warpType].coaddInputs.ccds,
skyInfo.wcs,
self.config.coaddPsf.makeControl()))
else:
if not self.config.doWriteEmptyWarps:
# No good pixels. Exposure still empty
coaddTempExps[warpType] = None
result = pipeBase.Struct(exposures=coaddTempExps)
return result
def run(self, patchRef):
"""Produce <coaddName>Coadd_tempExp images and (optional) <coaddName>Coadd_initPsf
<coaddName>Coadd_tempExp are produced by PSF-matching (optional) and warping.
If PSF-matching is used then <coaddName>Coadd_initPsf is also computed.
PSF matching is to a double gaussian model with core FWHM = self.config.warpAndPsfMatch.desiredFwhm
and wings of amplitude 1/10 of core and FWHM = 2.5 * core.
PSF-matching is performed before warping so the code can use the PSF models
associated with the calibrated science exposures (without having to warp those models).
@param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
plus the camera-specific filter key (e.g. "filter" or "band")
@return: a pipeBase.Struct with fields:
- dataRefList: a list of data references for the new <coaddName>Coadd_tempExp
@warning: this task assumes that all exposures in a coaddTempExp have the same filter.
@warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
(assembleCoadd should determine zeropoint scaling without referring to it).
"""
skyInfo = self.getSkyInfo(patchRef)
tractWcs = skyInfo.wcs
patchBBox = skyInfo.bbox
calExpRefList = self.selectExposures(patchRef=patchRef, wcs=tractWcs, bbox=patchBBox)
# initialize outputs
dataRefList = []
numExp = len(calExpRefList)
if numExp < 1:
raise pipeBase.TaskError("No exposures to coadd")
self.log.info("Process %s calexp" % (numExp,))
doPsfMatch = self.config.warpAndPsfMatch.desiredFwhm is not None
if not doPsfMatch:
self.log.info("No PSF matching will be done (desiredFwhm is None)")
tempExpName = self.config.coaddName + "Coadd_tempExp"
# compute tempKeyList: a tuple of ID key names in a calExpId that identify a coaddTempExp.
# You must also specify tract and patch to make a complete coaddTempExp ID.
butler = patchRef.butlerSubset.butler
tempExpKeySet = set(butler.getKeys(datasetType=tempExpName, level="Ccd")) - set(("patch", "tract"))
tempExpKeyList = tuple(sorted(tempExpKeySet))
# compute tempExpIdDict, a dict whose:
# - keys are tuples of coaddTempExp ID values in tempKeyList order
# - values are a list of calExp data references for calExp that belong in this coaddTempExp
tempExpIdDict = dict()
for calExpRef in calExpRefList:
calExpId = calExpRef.dataId
if not calExpRef.datasetExists("calexp"):
self.log.warn("Could not find calexp %s; skipping it" % (calExpId,))
continue
tempExpIdTuple = tuple(calExpId[key] for key in tempExpKeyList)
calExpSubsetRefList = tempExpIdDict.get(tempExpIdTuple)
if calExpSubsetRefList:
calExpSubsetRefList.append(calExpRef)
else:
tempExpIdDict[tempExpIdTuple] = [calExpRef]
numTempExp = len(tempExpIdDict)
coaddTempExp = None
for tempExpInd, calExpSubsetRefList in enumerate(tempExpIdDict.itervalues()):
# derive tempExpId from the first calExpId
tempExpId = dict((key, calExpSubsetRefList[0].dataId[key]) for key in tempExpKeyList)
tempExpId.update(patchRef.dataId)
tempExpRef = calExpRef.butlerSubset.butler.dataRef(
datasetType = tempExpName,
dataId = tempExpId,
)
if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=tempExpName):
self.log.info("tempCoaddExp exists %s; skipping it"%(tempExpId))
continue
self.log.info("Computing coaddTempExp %d of %d: id=%s" % (tempExpInd+1, numTempExp, tempExpId))
totGoodPix = 0
coaddTempExp = afwImage.ExposureF(patchBBox, tractWcs)
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
coaddTempExp.getMaskedImage().set(numpy.nan, edgeMask, numpy.inf)
didSetMetadata = False
for calExpInd, calExpRef in enumerate(calExpSubsetRefList):
self.log.info("Processing calexp %d of %d for this tempExp: id=%s" % \
(calExpInd+1, len(calExpSubsetRefList), calExpRef.dataId))
try:
exposure = self.warpAndPsfMatch.getCalExp(calExpRef, getPsf=doPsfMatch, bgSubtracted=self.config.bgSubtracted)
exposure = self.warpAndPsfMatch.run(exposure, wcs=tractWcs, maxBBox=patchBBox).exposure
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self._badPixelMask)
totGoodPix += numGoodPix
if numGoodPix == 0:
self.log.warn("Calexp %s has no good pixels in this patch" % (calExpRef.dataId,))
else:
self.log.info("Calexp %s has %s good pixels in this patch" % \
(calExpRef.dataId, numGoodPix))
if not didSetMetadata:
coaddTempExp.setCalib(exposure.getCalib())
coaddTempExp.setFilter(exposure.getFilter())
didSetMetadata = True
except Exception, e:
self.log.warn("Error processing calexp %s; skipping it: %s" % \
(calExpRef.dataId, e))
continue
if (totGoodPix == 0) or not didSetMetadata: # testing didSetMetadata is not needed but safer
self.log.warn("Could not compute coaddTempExp %s: no good pixels" % (tempExpRef.dataId,))
continue
self.log.info("coaddTempExp %s has %s good pixels" % (tempExpRef.dataId, totGoodPix))
if self.config.doWrite and coaddTempExp is not None:
self.log.info("Persisting %s %s" % (tempExpName, tempExpRef.dataId))
tempExpRef.put(coaddTempExp, tempExpName)
if self.config.warpAndPsfMatch.desiredFwhm is not None:
psfName = self.config.coaddName + "Coadd_initPsf"
self.log.info("Persisting %s %s" % (psfName, tempExpRef.dataId))
wcs = coaddTempExp.getWcs()
fwhmPixels = self.config.warpAndPsfMatch.desiredFwhm / wcs.pixelScale().asArcseconds()
kernelSize = int(round(fwhmPixels * self.config.coaddKernelSizeFactor))
kernelDim = afwGeom.Point2I(kernelSize, kernelSize)
coaddPsf = self.makeModelPsf(fwhmPixels=fwhmPixels, kernelDim=kernelDim)
patchRef.put(coaddPsf, psfName)
if coaddTempExp:
dataRefList.append(tempExpRef)
else:
self.log.warn("This %s temp coadd exposure could not be created"%(tempExpRef.dataId,))
