def setUp(self):
np.random.seed(10)
self.config = MakeCoaddTempExpConfig()
meanCalibration = 1e-4
calibrationErr = 1e-5
self.exposurePhotoCalib = lsst.afw.image.PhotoCalib(
meanCalibration, calibrationErr)
# An external photoCalib calibration to return
self.externalPhotoCalib = lsst.afw.image.PhotoCalib(1e-6, 1e-8)
crpix = lsst.geom.Point2D(0, 0)
crval = lsst.geom.SpherePoint(0, 45, lsst.geom.degrees)
cdMatrix = lsst.afw.geom.makeCdMatrix(scale=1.0 * lsst.geom.arcseconds)
self.skyWcs = lsst.afw.geom.makeSkyWcs(crpix, crval, cdMatrix)
externalCdMatrix = lsst.afw.geom.makeCdMatrix(scale=0.9 *
lsst.geom.arcseconds)
# An external skyWcs to return
self.externalSkyWcs = lsst.afw.geom.makeSkyWcs(crpix, crval,
externalCdMatrix)
self.exposure = lsst.afw.image.ExposureF(10, 10)
self.exposure.maskedImage.image.array = np.random.random(
(10, 10)).astype(np.float32) * 1000
self.exposure.maskedImage.variance.array = np.random.random(
(10, 10)).astype(np.float32)
# mask at least one pixel
self.exposure.maskedImage.mask[5, 5] = 3
# set the PhotoCalib and Wcs objects of this exposure.
self.exposure.setPhotoCalib(
lsst.afw.image.PhotoCalib(meanCalibration, calibrationErr))
self.exposure.setWcs(self.skyWcs)
# set to True in a test to raise NoResults for get('calexp')
self.raiseOnGetCalexp = False
def mockGet(datasetType, dataId=None, immediate=None):
"""Minimally fake a butler.get()."""
if "calexp" in datasetType:
if self.raiseOnGetCalexp:
raise NoResults("failed!", datasetType, dataId)
else:
return self.exposure.clone()
if "jointcal_photoCalib" in datasetType:
return self.externalPhotoCalib
if "fgcm_photoCalib" in datasetType:
return self.externalPhotoCalib
if "fgcm_tract_photoCalib" in datasetType:
return self.externalPhotoCalib
if "jointcal_wcs" in datasetType:
return self.externalSkyWcs
self.dataRef = unittest.mock.Mock(spec=ButlerDataRef)
self.dataRef.get.side_effect = mockGet
self.dataRef.dataId = {"ccd": 10000, "visit": 1}
mask = np.reshape(
np.reshape(mask_old, (-1, 1))[0:2200 * 2200], (2200, 2200))
variance = np.reshape(
np.reshape(variance_old, (-1, 1))[0:2200 * 2200], (2200, 2200))
sys.stderr.write("mask[0,0] = " + str(mask[0, 0]) +
", mask.dtype.type = " + str(mask.dtype.type) + "\n")
sys.stderr.write("variance[0,0] = " + str(variance[0, 0]) +
", variance.dtype.type = " +
str(variance.dtype.type) + "\n")
# mask = np.ones((MAX_X,MAX_Y), dtype=np.uint16)
# variance = np.ones((MAX_X,MAX_Y), dtype=np.float32)
maskedImage = bu.makeMaskedImageFromArrays(data, mask, variance)
image = afwImage.ExposureF(maskedImage)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
modelPsf = makeCTEConfig.modelPsf.apply()
image.setPsf(modelPsf)
detRes = detectCoaddSources.runDetection(image, idFactory=None)
#persist to SciDB
print(0)
sys.stdout.flush()
else:
sys.stderr.write("=====> DFZ DEBUG: pid " + str(pid) +
" finished at " + time.ctime() + "\n")
sys.stderr.write("=====> DFZ DEBUG: total run time = " +
str(datetime.datetime.now() - tm_start) +
" seconds\n")
def main():
# try out one exposure
#visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"]
visits = ["0288976", "0288935"]
ccds = []
exit(0)
for i in range(1, 61):
ccds.append(i)
filterName = 'g'
DATA_PATH = "/root/extra_home/lsst_data/"
#spathprefix = "/home/dongfang/download/lsst_data/"
spathprefix = DATA_PATH + "raw/"
#calexpsloc = "/home/dongfang/download/lsst_data/calexps/"
calexpsloc = DATA_PATH + "calexps/"
#coaddloc = "/home/dongfang/download/lsst_data/coadds/"
coaddloc = DATA_PATH + "coadds/"
#mergecoaddloc = "/home/dongfang/download/lsst_data/merge/"
mergecoaddloc = DATA_PATH + "merge/"
# Characterize Image
charImageConfig = CharacterizeImageConfig()
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(
projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333,
rotation=0.0,
patchBorder=100,
raList=[154.10660740464786],
tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
coaddTempDict = {}
calibResDict = {}
f = open("log.txt", 'wb')
start = datetime.datetime.now()
#process CCDs to create calexps.
for v in visits:
for ccd in ccds:
visit = int(v)
filename = "instcal" + v + "." + str(ccd) + ".fits"
calexpfn = calexpsloc + v + "/" + filename
source = spathprefix + v + "/" + filename
exposure = afwImg.ExposureF(source)
try:
# Characterize Image
charRes = charImage.characterize(exposure,
exposureIdInfo=None,
background=None)
except:
f.write("DFZ DEBUG at charRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
# Caliberate Image
calibRes = calibrateTask.calibrate(
charRes.exposure,
exposureIdInfo=None,
background=charRes.background,
icSourceCat=None)
except:
f.write("DFZ DEBUG at calibRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
#write out calexps
calibRes.exposure.writeFits(calexpfn)
#calbresDict.append((v,ccd),calibRes)
except:
f.write("DFZ DEBUG at calibRes.exposure: errors in visit " +
v + ", ccd " + str(ccd) + "\n")
end = datetime.datetime.now()
d = end - start
f.write("time for creating calexps: ")
f.write(str(d.total_seconds()))
f.write("\n")
#time for creating co-add tempexps.
start = datetime.datetime.now()
# map calexps to patch-ids
visit = visits[0]
ccdsPerPatch = []
for ccd in ccds:
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
exposure = afwImg.ExposureF(source)
bbox = exposure.getBBox()
wcs = exposure.getWcs()
corners = bbox.getCorners()
xIndexMax, yIndexMax = tract.findPatch(
wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex()
xIndexMin, yIndexMin = tract.findPatch(
wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex()
yy = range(yIndexMin, yIndexMax + 1)
xx = range(xIndexMin, xIndexMax + 1)
for yIdx in yy:
for xIdx in xx:
ccdsPerPatch.append((ccd, (xIdx, yIdx)))
print len(ccdsPerPatch)
#import cPickle
#cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch)
# import cPickle
# f = open("ccdsInPatch.p",'wb')
# cPickle.dump(ccdsInPatch,f)
#import cPickle
#ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb'))
df = pd.DataFrame(ccdsPerPatch)
dfgby = df.groupby(1)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
coaddTempExpDict = {}
for visit in visits:
for a in dfgby.indices:
coaddTempExpDict[a] = {}
xInd = a[0]
yInd = a[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(
numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby.get_group(a)[0].ravel():
print a
print b
if not setInputRecorder:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
try:
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
except:
f.write("DFZ DEBUG at inputRecorder\n")
setInputRecorder = True
numGoodPix = 0
ccd = b
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
calExp = afwImg.ExposureF(source)
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(
modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
coaddTempExpDict[a][v] = coaddTempExp
coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str(
xInd) + "_" + str(yInd) + ".fits"
coaddTempExp.writeFits(coaddfilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating co-add tempexps:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
#DFZ: stop here
exit(0)
start = datetime.datetime.now()
config = AssembleCoaddConfig()
assembleTask = AssembleCoaddTask(config=config)
mergcoadds = {}
for a in dfgby.indices:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
xInd = a[0]
yInd = a[1]
imageScalerRes = prepareInputs(coaddTempExpDict[a].values(),
coaddTempExpDict[a].keys(),
assembleTask)
mask = None
doClip = False
if mask is None:
mask = assembleTask.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip)
statsCtrl.setNumIter(assembleTask.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
for plane, threshold in assembleTask.config.maskPropagationThresholds.items(
):
bit = afwImage.MaskU.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
if doClip:
statsFlags = afwMath.MEANCLIP
else:
statsFlags = afwMath.MEAN
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib())
coaddExposure.getInfo().setCoaddInputs(
assembleTask.inputRecorder.makeCoaddInputs())
#remember to set metadata if you want any hope of running detection and measurement on this coadd:
#self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
#most important thing is the psf
coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter())
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
for tempExp, weight in zip(coaddTempExpDict[a].values(),
imageScalerRes.weightList):
assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp,
weight)
#takes numCcds as argument
coaddInputs.ccds.reserve(ccdsinPatch)
coaddInputs.visits.reserve(len(imageScalerRes.dataIdList))
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
coaddExposure.setPsf(psf)
maskedImageList = afwImage.vectorMaskedImageF()
coaddMaskedImage = coaddExposure.getMaskedImage()
for dataId, imageScaler, exposure in zip(
imageScalerRes.dataIdList, imageScalerRes.imageScalerList,
coaddTempExpDict[a].values()):
print dataId, imageScaler, exposure
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
maskedImageList.append(maskedImage)
maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags,
statsCtrl,
imageScalerRes.weightList)
coaddMaskedImage.assign(maskedImage, skyInfo.bbox)
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(),
coaddMaskedImage.getVariance())
# write out Coadd!
mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits"
mergcoadds[a] = coaddExposure
coaddExposure.writeFits(mergefilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating merged co-adds:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
start = datetime.datetime.now()
config = DetectCoaddSourcesConfig()
detectCoaddSources = DetectCoaddSourcesTask(config=config)
for a in dfgby.indices:
# Detect on Coadd
exp = mergcoadds[a]
detRes = detectCoaddSources.runDetection(exp, idFactory=None)
end = datetime.datetime.now()
d = end - start
f.write("time for detecting sources:\n ")
f.write(str(d.total_seconds()))
f.close()
def makePatch(patchId, visit, dfgby_raveled, ccdsNeeded, hits_skymap):
import lsst.afw.image as afwImg
from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask, MakeCoaddTempExpConfig
import lsst.coadd.utils as coaddUtils
import numpy
import lsst.skymap as skymap
from lsst.meas.algorithms import CoaddPsf
import lsst.skymap as skymap
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
xInd = patchId[0]
yInd = patchId[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImg.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(numpy.nan,
afwImg.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby_raveled: # dfgby.get_group(a)[0].ravel():
if not setInputRecorder:
ccdsinPatch = len(
dfgby_raveled) # len(dfgby.get_group(a)[0].ravel())
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
setInputRecorder = True
numGoodPix = 0
ccd = b
if ccdsNeeded[(v, ccd)] is not None:
calExp = ccdsNeeded[(v, ccd)].exposure
else:
continue
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp, modelPsf=modelPsf, wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(),
makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
return coaddTempExp