def finish(self, coaddTempExp, nGoodPix=None):
"""Finish creating the CoaddInputs for a CoaddTempExp.
@param[in,out] coaddTempExp Exposure object from which to obtain the PSF, WCS, and bounding
box for the entry in the 'visits' table. On return, the completed
CoaddInputs object will be attached to it.
@param[in] nGoodPix Total number of good pixels in the CoaddTempExp; ignored unless
saveVisitGoodPix is true.
"""
self._setExposureInfoInRecord(exposure=coaddTempExp, record=self.visitRecord)
if self.task.config.saveVisitGoodPix:
self.visitRecord.setI(self.task.visitGoodPixKey, nGoodPix)
coaddTempExp.getInfo().setCoaddInputs(self.coaddInputs)
wcs = coaddTempExp.getWcs()
if False:
# This causes a test failure, pending fix in issue HSC-802
coaddTempExp.setPsf(CoaddPsf(self.coaddInputs.ccds, wcs))
apCorrMap = makeCoaddApCorrMap(self.coaddInputs.ccds, coaddTempExp.getBBox(afwImage.PARENT), wcs)
coaddTempExp.getInfo().setApCorrMap(apCorrMap)
def run(self, exposure, sensorRef, templateIdList=None):
"""Retrieve and mosaic a template coadd that overlaps the exposure where
the template spans multiple tracts.
The resulting template image will be an average of all the input templates from
the separate tracts.
The PSF on the template is created by combining the CoaddPsf on each template image
into a meta-CoaddPsf.
Parameters
----------
exposure: `lsst.afw.image.Exposure`
an exposure for which to generate an overlapping template
sensorRef : TYPE
a Butler data reference that can be used to obtain coadd data
templateIdList : TYPE, optional
list of data ids (unused)
Returns
-------
result : `struct`
return a pipeBase.Struct:
- ``exposure`` : a template coadd exposure assembled out of patches
- ``sources`` : None for this subtask
"""
# Table for CoaddPSF
tractsSchema = afwTable.ExposureTable.makeMinimalSchema()
tractKey = tractsSchema.addField('tract',
type=np.int32,
doc='Which tract')
patchKey = tractsSchema.addField('patch',
type=np.int32,
doc='Which patch')
weightKey = tractsSchema.addField(
'weight', type=float, doc='Weight for each tract, should be 1')
tractsCatalog = afwTable.ExposureCatalog(tractsSchema)
skyMap = sensorRef.get(datasetType=self.config.coaddName +
"Coadd_skyMap")
expWcs = exposure.getWcs()
expBoxD = geom.Box2D(exposure.getBBox())
expBoxD.grow(self.config.templateBorderSize)
ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
centralTractInfo = skyMap.findTract(ctrSkyPos)
if not centralTractInfo:
raise RuntimeError("No suitable tract found for central point")
self.log.info("Central skyMap tract %s" % (centralTractInfo.getId(), ))
skyCorners = [
expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()
]
tractPatchList = skyMap.findTractPatchList(skyCorners)
if not tractPatchList:
raise RuntimeError("No suitable tract found")
self.log.info("All overlapping skyMap tracts %s" %
([a[0].getId() for a in tractPatchList]))
# Move central tract to front of the list and use as the reference
tracts = [tract[0].getId() for tract in tractPatchList]
centralIndex = tracts.index(centralTractInfo.getId())
tracts.insert(0, tracts.pop(centralIndex))
tractPatchList.insert(0, tractPatchList.pop(centralIndex))
coaddPsf = None
coaddFilter = None
nPatchesFound = 0
maskedImageList = []
weightList = []
for itract, tract in enumerate(tracts):
tractInfo = tractPatchList[itract][0]
coaddWcs = tractInfo.getWcs()
coaddBBox = geom.Box2D()
for skyPos in skyCorners:
coaddBBox.include(coaddWcs.skyToPixel(skyPos))
coaddBBox = geom.Box2I(coaddBBox)
if itract == 0:
# Define final wcs and bounding box from the reference tract
finalWcs = coaddWcs
finalBBox = coaddBBox
patchList = tractPatchList[itract][1]
for patchInfo in patchList:
self.log.info('Adding patch %s from tract %s' %
(patchInfo.getIndex(), tract))
# Local patch information
patchSubBBox = geom.Box2I(patchInfo.getInnerBBox())
patchSubBBox.clip(coaddBBox)
patchInt = int(
f"{patchInfo.getIndex()[0]}{patchInfo.getIndex()[1]}")
innerBBox = geom.Box2I(tractInfo._minimumBoundingBox(finalWcs))
if itract == 0:
# clip to image and tract boundaries
patchSubBBox.clip(finalBBox)
patchSubBBox.clip(innerBBox)
if patchSubBBox.getArea() == 0:
self.log.debug("No ovlerap for patch %s" % patchInfo)
continue
patchArgDict = dict(
datasetType="deepCoadd_sub",
bbox=patchSubBBox,
tract=tractInfo.getId(),
patch="%s,%s" %
(patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
filter=exposure.getFilter().getName())
coaddPatch = sensorRef.get(**patchArgDict)
if coaddFilter is None:
coaddFilter = coaddPatch.getFilter()
# create full image from final bounding box
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(coaddPatch.maskedImage,
patchSubBBox)
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(coaddPatch.getPsf())
record.setWcs(coaddPatch.getWcs())
record.setPhotoCalib(coaddPatch.getPhotoCalib())
record.setBBox(patchSubBBox)
record.set(tractKey, tract)
record.set(patchKey, patchInt)
record.set(weightKey, 1.)
nPatchesFound += 1
else:
# compute the exposure bounding box in a tract that is not the reference tract
localBox = geom.Box2I()
for skyPos in skyCorners:
localBox.include(
geom.Point2I(
tractInfo.getWcs().skyToPixel(skyPos)))
# clip to patch bounding box
localBox.clip(patchSubBBox)
# grow border to deal with warping at edges
localBox.grow(self.config.templateBorderSize)
# clip to tract inner bounding box
localInnerBBox = geom.Box2I(
tractInfo._minimumBoundingBox(tractInfo.getWcs()))
localBox.clip(localInnerBBox)
patchArgDict = dict(
datasetType="deepCoadd_sub",
bbox=localBox,
tract=tractInfo.getId(),
patch="%s,%s" %
(patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
filter=exposure.getFilter().getName())
coaddPatch = sensorRef.get(**patchArgDict)
# warp to reference tract wcs
xyTransform = afwGeom.makeWcsPairTransform(
coaddPatch.getWcs(), finalWcs)
psfWarped = WarpedPsf(coaddPatch.getPsf(), xyTransform)
warped = self.warper.warpExposure(finalWcs,
coaddPatch,
maxBBox=finalBBox)
# check if warpped image is viable
if warped.getBBox().getArea() == 0:
self.log.info(
"No ovlerap for warped patch %s. Skipping" %
patchInfo)
continue
warped.setPsf(psfWarped)
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(warped.maskedImage,
warped.getBBox())
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(psfWarped)
record.setWcs(finalWcs)
record.setPhotoCalib(coaddPatch.getPhotoCalib())
record.setBBox(warped.getBBox())
record.set(tractKey, tract)
record.set(patchKey, patchInt)
record.set(weightKey, 1.)
nPatchesFound += 1
if nPatchesFound == 0:
raise RuntimeError("No patches found!")
# Combine images from individual patches together
# Do not mask any values
statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
maskMap = []
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
coaddExposure = afwImage.ExposureF(finalBBox, finalWcs)
coaddExposure.maskedImage.set(np.nan,
afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
xy0 = coaddExposure.getXY0()
coaddExposure.maskedImage = afwMath.statisticsStack(
maskedImageList, statsFlags, statsCtrl, weightList, 0, maskMap)
coaddExposure.maskedImage.setXY0(xy0)
coaddPsf = CoaddPsf(tractsCatalog, finalWcs,
self.config.coaddPsf.makeControl())
if coaddPsf is None:
raise RuntimeError("No coadd Psf found!")
coaddExposure.setPsf(coaddPsf)
coaddExposure.setFilter(coaddFilter)
return pipeBase.Struct(exposure=coaddExposure, sources=None)
def createTempExp(self, calexpRefList, skyInfo, visitId=0):
"""Create a tempExp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp ("tempExp") of that exposure to the supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return warped exposure, or None if no pixels overlap
"""
inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None
for calExpInd, calExpRef in enumerate(calexpRefList):
self.log.info("Processing calexp %d of %d for this tempExp: id=%s",
calExpInd+1, len(calexpRefList), calExpRef.dataId)
try:
ccdId = calExpRef.get("ccdExposureId", immediate=True)
except Exception:
ccdId = calExpInd
numGoodPix = 0
try:
# We augment the dataRef here with the tract, which is harmless for loading things
# like calexps that don't need the tract, and necessary for meas_mosaic outputs,
# which do.
calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
tract=skyInfo.tractInfo.getId())
calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix += numGoodPix
self.log.debug("Calexp %s has %d good pixels in this patch (%.1f%%)",
calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea())
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(exposure.getCalib())
coaddTempExp.setFilter(exposure.getFilter())
# PSF replaced with CoaddPsf after loop if and only if creating direct warp
coaddTempExp.setPsf(exposure.getPsf())
didSetMetadata = True
except Exception as e:
self.log.warn("Error processing calexp %s; skipping it: %s", calExpRef.dataId, e)
continue
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata and not self.config.doPsfMatch:
coaddTempExp.setPsf(CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs))
self.log.info("coaddTempExp has %d good pixels (%.1f%%)",
totGoodPix, 100.0*totGoodPix/skyInfo.bbox.getArea())
return coaddTempExp if totGoodPix > 0 and didSetMetadata else None
def main():
# try out one exposure
#visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"]
visits = ["0288976", "0288935"]
ccds = []
exit(0)
for i in range(1, 61):
ccds.append(i)
filterName = 'g'
DATA_PATH = "/root/extra_home/lsst_data/"
#spathprefix = "/home/dongfang/download/lsst_data/"
spathprefix = DATA_PATH + "raw/"
#calexpsloc = "/home/dongfang/download/lsst_data/calexps/"
calexpsloc = DATA_PATH + "calexps/"
#coaddloc = "/home/dongfang/download/lsst_data/coadds/"
coaddloc = DATA_PATH + "coadds/"
#mergecoaddloc = "/home/dongfang/download/lsst_data/merge/"
mergecoaddloc = DATA_PATH + "merge/"
# Characterize Image
charImageConfig = CharacterizeImageConfig()
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(
projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333,
rotation=0.0,
patchBorder=100,
raList=[154.10660740464786],
tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
coaddTempDict = {}
calibResDict = {}
f = open("log.txt", 'wb')
start = datetime.datetime.now()
#process CCDs to create calexps.
for v in visits:
for ccd in ccds:
visit = int(v)
filename = "instcal" + v + "." + str(ccd) + ".fits"
calexpfn = calexpsloc + v + "/" + filename
source = spathprefix + v + "/" + filename
exposure = afwImg.ExposureF(source)
try:
# Characterize Image
charRes = charImage.characterize(exposure,
exposureIdInfo=None,
background=None)
except:
f.write("DFZ DEBUG at charRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
# Caliberate Image
calibRes = calibrateTask.calibrate(
charRes.exposure,
exposureIdInfo=None,
background=charRes.background,
icSourceCat=None)
except:
f.write("DFZ DEBUG at calibRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
#write out calexps
calibRes.exposure.writeFits(calexpfn)
#calbresDict.append((v,ccd),calibRes)
except:
f.write("DFZ DEBUG at calibRes.exposure: errors in visit " +
v + ", ccd " + str(ccd) + "\n")
end = datetime.datetime.now()
d = end - start
f.write("time for creating calexps: ")
f.write(str(d.total_seconds()))
f.write("\n")
#time for creating co-add tempexps.
start = datetime.datetime.now()
# map calexps to patch-ids
visit = visits[0]
ccdsPerPatch = []
for ccd in ccds:
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
exposure = afwImg.ExposureF(source)
bbox = exposure.getBBox()
wcs = exposure.getWcs()
corners = bbox.getCorners()
xIndexMax, yIndexMax = tract.findPatch(
wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex()
xIndexMin, yIndexMin = tract.findPatch(
wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex()
yy = range(yIndexMin, yIndexMax + 1)
xx = range(xIndexMin, xIndexMax + 1)
for yIdx in yy:
for xIdx in xx:
ccdsPerPatch.append((ccd, (xIdx, yIdx)))
print len(ccdsPerPatch)
#import cPickle
#cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch)
# import cPickle
# f = open("ccdsInPatch.p",'wb')
# cPickle.dump(ccdsInPatch,f)
#import cPickle
#ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb'))
df = pd.DataFrame(ccdsPerPatch)
dfgby = df.groupby(1)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
coaddTempExpDict = {}
for visit in visits:
for a in dfgby.indices:
coaddTempExpDict[a] = {}
xInd = a[0]
yInd = a[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(
numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby.get_group(a)[0].ravel():
print a
print b
if not setInputRecorder:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
try:
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
except:
f.write("DFZ DEBUG at inputRecorder\n")
setInputRecorder = True
numGoodPix = 0
ccd = b
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
calExp = afwImg.ExposureF(source)
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(
modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
coaddTempExpDict[a][v] = coaddTempExp
coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str(
xInd) + "_" + str(yInd) + ".fits"
coaddTempExp.writeFits(coaddfilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating co-add tempexps:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
#DFZ: stop here
exit(0)
start = datetime.datetime.now()
config = AssembleCoaddConfig()
assembleTask = AssembleCoaddTask(config=config)
mergcoadds = {}
for a in dfgby.indices:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
xInd = a[0]
yInd = a[1]
imageScalerRes = prepareInputs(coaddTempExpDict[a].values(),
coaddTempExpDict[a].keys(),
assembleTask)
mask = None
doClip = False
if mask is None:
mask = assembleTask.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip)
statsCtrl.setNumIter(assembleTask.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
for plane, threshold in assembleTask.config.maskPropagationThresholds.items(
):
bit = afwImage.MaskU.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
if doClip:
statsFlags = afwMath.MEANCLIP
else:
statsFlags = afwMath.MEAN
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib())
coaddExposure.getInfo().setCoaddInputs(
assembleTask.inputRecorder.makeCoaddInputs())
#remember to set metadata if you want any hope of running detection and measurement on this coadd:
#self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
#most important thing is the psf
coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter())
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
for tempExp, weight in zip(coaddTempExpDict[a].values(),
imageScalerRes.weightList):
assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp,
weight)
#takes numCcds as argument
coaddInputs.ccds.reserve(ccdsinPatch)
coaddInputs.visits.reserve(len(imageScalerRes.dataIdList))
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
coaddExposure.setPsf(psf)
maskedImageList = afwImage.vectorMaskedImageF()
coaddMaskedImage = coaddExposure.getMaskedImage()
for dataId, imageScaler, exposure in zip(
imageScalerRes.dataIdList, imageScalerRes.imageScalerList,
coaddTempExpDict[a].values()):
print dataId, imageScaler, exposure
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
maskedImageList.append(maskedImage)
maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags,
statsCtrl,
imageScalerRes.weightList)
coaddMaskedImage.assign(maskedImage, skyInfo.bbox)
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(),
coaddMaskedImage.getVariance())
# write out Coadd!
mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits"
mergcoadds[a] = coaddExposure
coaddExposure.writeFits(mergefilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating merged co-adds:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
start = datetime.datetime.now()
config = DetectCoaddSourcesConfig()
detectCoaddSources = DetectCoaddSourcesTask(config=config)
for a in dfgby.indices:
# Detect on Coadd
exp = mergcoadds[a]
detRes = detectCoaddSources.runDetection(exp, idFactory=None)
end = datetime.datetime.now()
d = end - start
f.write("time for detecting sources:\n ")
f.write(str(d.total_seconds()))
f.close()
def makePatch(patchId, visit, dfgby_raveled, ccdsNeeded, hits_skymap):
import lsst.afw.image as afwImg
from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask, MakeCoaddTempExpConfig
import lsst.coadd.utils as coaddUtils
import numpy
import lsst.skymap as skymap
from lsst.meas.algorithms import CoaddPsf
import lsst.skymap as skymap
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
xInd = patchId[0]
yInd = patchId[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImg.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan,
afwImg.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby_raveled: # dfgby.get_group(a)[0].ravel():
if not setInputRecorder:
ccdsinPatch = len(
dfgby_raveled) # len(dfgby.get_group(a)[0].ravel())
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
setInputRecorder = True
numGoodPix = 0
ccd = b
if ccdsNeeded[(v, ccd)] is not None:
calExp = ccdsNeeded[(v, ccd)].exposure
else:
continue
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(),
makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
return coaddTempExp
def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
"""Warp coadds from multiple tracts to form a template for image diff.
Where the tracts overlap, the resulting template image is averaged.
The PSF on the template is created by combining the CoaddPsf on each
template image into a meta-CoaddPsf.
Parameters
----------
coaddExposures : `list` of `lsst.afw.image.Exposure`
Coadds to be mosaicked
bbox : `lsst.geom.Box2I`
Template Bounding box of the detector geometry onto which to
resample the coaddExposures
wcs : `lsst.afw.geom.SkyWcs`
Template WCS onto which to resample the coaddExposures
dataIds : `list` of `lsst.daf.butler.DataCoordinate`
Record of the tract and patch of each coaddExposure.
**kwargs
Any additional keyword parameters.
Returns
-------
result : `lsst.pipe.base.Struct` containing
- ``outputExposure`` : a template coadd exposure assembled out of patches
"""
# Table for CoaddPSF
tractsSchema = afwTable.ExposureTable.makeMinimalSchema()
tractKey = tractsSchema.addField('tract',
type=np.int32,
doc='Which tract')
patchKey = tractsSchema.addField('patch',
type=np.int32,
doc='Which patch')
weightKey = tractsSchema.addField(
'weight', type=float, doc='Weight for each tract, should be 1')
tractsCatalog = afwTable.ExposureCatalog(tractsSchema)
finalWcs = wcs
bbox.grow(self.config.templateBorderSize)
finalBBox = bbox
nPatchesFound = 0
maskedImageList = []
weightList = []
for coaddExposure, dataId in zip(coaddExposures, dataIds):
# warp to detector WCS
warped = self.warper.warpExposure(finalWcs,
coaddExposure,
maxBBox=finalBBox)
# Check if warped image is viable
if not np.any(np.isfinite(warped.image.array)):
self.log.info("No overlap for warped %s. Skipping" % dataId)
continue
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(np.nan,
afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(coaddExposure.getPsf())
record.setWcs(coaddExposure.getWcs())
record.setPhotoCalib(coaddExposure.getPhotoCalib())
record.setBBox(coaddExposure.getBBox())
record.setValidPolygon(
afwGeom.Polygon(
geom.Box2D(coaddExposure.getBBox()).getCorners()))
record.set(tractKey, dataId['tract'])
record.set(patchKey, dataId['patch'])
record.set(weightKey, 1.)
nPatchesFound += 1
if nPatchesFound == 0:
raise pipeBase.NoWorkFound("No patches found to overlap detector")
# Combine images from individual patches together
statsFlags = afwMath.stringToStatisticsProperty('MEAN')
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
templateExposure = afwImage.ExposureF(finalBBox, finalWcs)
templateExposure.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
xy0 = templateExposure.getXY0()
# Do not mask any values
templateExposure.maskedImage = afwMath.statisticsStack(maskedImageList,
statsFlags,
statsCtrl,
weightList,
clipped=0,
maskMap=[])
templateExposure.maskedImage.setXY0(xy0)
# CoaddPsf centroid not only must overlap image, but must overlap the part of
# image with data. Use centroid of region with data
boolmask = templateExposure.mask.array & templateExposure.mask.getPlaneBitMask(
'NO_DATA') == 0
maskx = afwImage.makeMaskFromArray(boolmask.astype(afwImage.MaskPixel))
centerCoord = afwGeom.SpanSet.fromMask(maskx, 1).computeCentroid()
ctrl = self.config.coaddPsf.makeControl()
coaddPsf = CoaddPsf(tractsCatalog, finalWcs, centerCoord,
ctrl.warpingKernelName, ctrl.cacheSize)
if coaddPsf is None:
raise RuntimeError("CoaddPsf could not be constructed")
templateExposure.setPsf(coaddPsf)
templateExposure.setFilter(coaddExposure.getFilter())
templateExposure.setPhotoCalib(coaddExposure.getPhotoCalib())
return pipeBase.Struct(outputExposure=templateExposure)
def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None):
"""Create a Warp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp (previously called a coaddTempExp) of that exposure to the
supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return a pipeBase Struct containing:
- exposures: a dictionary containing the warps requested:
"direct": direct warp if config.makeDirect
"psfMatched": PSF-matched warp if config.makePsfMatched
"""
warpTypeList = self.getWarpTypeList()
totGoodPix = {warpType: 0 for warpType in warpTypeList}
didSetMetadata = {warpType: False for warpType in warpTypeList}
coaddTempExps = {
warpType: self._prepareEmptyExposure(skyInfo)
for warpType in warpTypeList
}
inputRecorder = {
warpType: self.inputRecorder.makeCoaddTempExpRecorder(
visitId, len(calExpList))
for warpType in warpTypeList
}
modelPsf = self.config.modelPsf.apply(
) if self.config.makePsfMatched else None
if dataIdList is None:
dataIdList = ccdIdList
for calExpInd, (calExp, ccdId, dataId) in enumerate(
zip(calExpList, ccdIdList, dataIdList)):
self.log.info("Processing calexp %d of %d for this Warp: id=%s",
calExpInd + 1, len(calExpList), dataId)
try:
warpedAndMatched = self.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox,
makeDirect=self.config.makeDirect,
makePsfMatched=self.config.makePsfMatched)
except Exception as e:
self.log.warn(
"WarpAndPsfMatch failed for calexp %s; skipping it: %s",
dataId, e)
continue
try:
numGoodPix = {warpType: 0 for warpType in warpTypeList}
for warpType in warpTypeList:
exposure = warpedAndMatched.getDict()[warpType]
if exposure is None:
continue
coaddTempExp = coaddTempExps[warpType]
if didSetMetadata[warpType]:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getPhotoCalib(
).getInstFluxAtZeroMagnitude() / exposure.
getPhotoCalib().getInstFluxAtZeroMagnitude())
del mimg
numGoodPix[warpType] = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix[warpType] += numGoodPix[warpType]
self.log.debug(
"Calexp %s has %d good pixels in this patch (%.1f%%) for %s",
dataId, numGoodPix[warpType],
100.0 * numGoodPix[warpType] / skyInfo.bbox.getArea(),
warpType)
if numGoodPix[warpType] > 0 and not didSetMetadata[
warpType]:
coaddTempExp.setPhotoCalib(exposure.getPhotoCalib())
coaddTempExp.setFilter(exposure.getFilter())
coaddTempExp.getInfo().setVisitInfo(
exposure.getInfo().getVisitInfo())
# PSF replaced with CoaddPsf after loop if and only if creating direct warp
coaddTempExp.setPsf(exposure.getPsf())
didSetMetadata[warpType] = True
# Need inputRecorder for CoaddApCorrMap for both direct and PSF-matched
inputRecorder[warpType].addCalExp(calExp, ccdId,
numGoodPix[warpType])
except Exception as e:
self.log.warn("Error processing calexp %s; skipping it: %s",
dataId, e)
continue
for warpType in warpTypeList:
self.log.info(
"%sWarp has %d good pixels (%.1f%%)", warpType,
totGoodPix[warpType],
100.0 * totGoodPix[warpType] / skyInfo.bbox.getArea())
if totGoodPix[warpType] > 0 and didSetMetadata[warpType]:
inputRecorder[warpType].finish(coaddTempExps[warpType],
totGoodPix[warpType])
if warpType == "direct":
coaddTempExps[warpType].setPsf(
CoaddPsf(inputRecorder[warpType].coaddInputs.ccds,
skyInfo.wcs,
self.config.coaddPsf.makeControl()))
else:
if not self.config.doWriteEmptyWarps:
# No good pixels. Exposure still empty
coaddTempExps[warpType] = None
result = pipeBase.Struct(exposures=coaddTempExps)
return result
class MakeCoaddTempExpTask(CoaddBaseTask):
"""Task to produce <coaddName>Coadd_tempExp images
"""
ConfigClass = MakeCoaddTempExpConfig
_DefaultName = "makeCoaddTempExp"
def __init__(self, *args, **kwargs):
CoaddBaseTask.__init__(self, *args, **kwargs)
self.makeSubtask("warpAndPsfMatch")
@pipeBase.timeMethod
def run(self, patchRef, selectDataList=[]):
"""Produce <coaddName>Coadd_tempExp images
<coaddName>Coadd_tempExp are produced by PSF-matching (optional) and warping.
@param[in] patchRef: data reference for sky map patch. Must include keys "tract", "patch",
plus the camera-specific filter key (e.g. "filter" or "band")
@return: dataRefList: a list of data references for the new <coaddName>Coadd_tempExp
@warning: this task assumes that all exposures in a coaddTempExp have the same filter.
@warning: this task sets the Calib of the coaddTempExp to the Calib of the first calexp
with any good pixels in the patch. For a mosaic camera the resulting Calib should be ignored
(assembleCoadd should determine zeropoint scaling without referring to it).
"""
skyInfo = self.getSkyInfo(patchRef)
calExpRefList = self.selectExposures(patchRef, skyInfo, selectDataList=selectDataList)
if len(calExpRefList) == 0:
self.log.warn("No exposures to coadd for patch %s" % patchRef.dataId)
return None
self.log.info("Selected %d calexps for patch %s" % (len(calExpRefList), patchRef.dataId))
calExpRefList = [calExpRef for calExpRef in calExpRefList if calExpRef.datasetExists("calexp")]
self.log.info("Processing %d existing calexps for patch %s" % (len(calExpRefList), patchRef.dataId))
groupData = groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(),
self.getTempExpDatasetName())
self.log.info("Processing %d tempExps for patch %s" % (len(groupData.groups), patchRef.dataId))
dataRefList = []
for i, (tempExpTuple, calexpRefList) in enumerate(groupData.groups.iteritems()):
tempExpRef = getGroupDataRef(patchRef.getButler(), self.getTempExpDatasetName(),
tempExpTuple, groupData.keys)
if not self.config.doOverwrite and tempExpRef.datasetExists(datasetType=self.getTempExpDatasetName()):
self.log.info("tempCoaddExp %s exists; skipping" % (tempExpRef.dataId,))
dataRefList.append(tempExpRef)
continue
self.log.info("Processing tempExp %d/%d: id=%s" % (i, len(groupData.groups), tempExpRef.dataId))
# TODO: mappers should define a way to go from the "grouping keys" to a numeric ID (#2776).
# For now, we try to get a long integer "visit" key, and if we can't, we just use the index
# of the visit in the list.
try:
visitId = long(tempExpRef.dataId["visit"])
except (KeyError, ValueError):
visitId = i
exp = self.createTempExp(calexpRefList, skyInfo, visitId)
if exp is not None:
dataRefList.append(tempExpRef)
if self.config.doWrite:
self.writeCoaddOutput(tempExpRef, exp, "tempExp")
else:
self.log.warn("tempExp %s could not be created" % (tempExpRef.dataId,))
return dataRefList
def createTempExp(self, calexpRefList, skyInfo, visitId=0):
"""Create a tempExp from inputs
We iterate over the multiple calexps in a single exposure to construct
the warp ("tempExp") of that exposure to the supplied tract/patch.
Pixels that receive no pixels are set to NAN; this is not correct
(violates LSST algorithms group policy), but will be fixed up by
interpolating after the coaddition.
@param calexpRefList: List of data references for calexps that (may)
overlap the patch of interest
@param skyInfo: Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch
@param visitId: integer identifier for visit, for the table that will
produce the CoaddPsf
@return warped exposure, or None if no pixels overlap
"""
inputRecorder = self.inputRecorder.makeCoaddTempExpRecorder(visitId, len(calexpRefList))
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"), numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = self.config.modelPsf.apply() if self.config.doPsfMatch else None
for calExpInd, calExpRef in enumerate(calexpRefList):
self.log.info("Processing calexp %d of %d for this tempExp: id=%s" %
(calExpInd+1, len(calexpRefList), calExpRef.dataId))
try:
ccdId = calExpRef.get("ccdExposureId", immediate=True)
except Exception:
ccdId = calExpInd
numGoodPix = 0
try:
# We augment the dataRef here with the tract, which is harmless for loading things
# like calexps that don't need the tract, and necessary for meas_mosaic outputs,
# which do.
calExpRef = calExpRef.butlerSubset.butler.dataRef("calexp", dataId=calExpRef.dataId,
tract=skyInfo.tractInfo.getId())
calExp = self.getCalExp(calExpRef, bgSubtracted=self.config.bgSubtracted)
exposure = self.warpAndPsfMatch.run(calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = exposure.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] / exposure.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(), exposure.getMaskedImage(), self.getBadPixelMask())
totGoodPix += numGoodPix
self.log.logdebug("Calexp %s has %d good pixels in this patch (%.1f%%)" %
(calExpRef.dataId, numGoodPix, 100.0*numGoodPix/skyInfo.bbox.getArea()))
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(exposure.getCalib())
coaddTempExp.setFilter(exposure.getFilter())
didSetMetadata = True
except Exception, e:
self.log.warn("Error processing calexp %s; skipping it: %s" % (calExpRef.dataId, e))
continue
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(modelPsf if self.config.doPsfMatch else
CoaddPsf(inputRecorder.coaddInputs.ccds, skyInfo.wcs))
self.log.info("coaddTempExp has %d good pixels (%.1f%%)" %
(totGoodPix, 100.0*totGoodPix/skyInfo.bbox.getArea()))
return coaddTempExp if totGoodPix > 0 and didSetMetadata else None
