def writeFcr(self, dataRefList):
self.log.info("Write Fcr ...")
M_LN10 = math.log(10)
dmag = list()
for m in self.matchVec:
if (m.good == True and m.mag != -9999 and m.jstar != -1 and
m.mag0 != -9999 and m.mag_cat != -9999):
mag = m.mag
mag_cat = m.mag_cat
exp_cor = -2.5*math.log10(self.fexp[m.iexp])
chip_cor = -2.5*math.log10(self.fchip[m.ichip])
gain_cor = self.ffpSet[m.iexp].eval(m.u, m.v)
mag_cor = mag + exp_cor + chip_cor + gain_cor
dmag.append(mag_cor - mag_cat)
std, mean, n = mosaicUtils.clippedStd(numpy.array(dmag), 2.1)
for dataRef in dataRefList:
iexp = dataRef.dataId["visit"]
ichip = dataRef.dataId["ccd"]
try:
x0 = self.coeffSet[iexp].x0
y0 = self.coeffSet[iexp].y0
except:
x0 = 0.0
y0 = 0.0
newP = measMosaic.convertFluxFitParams(measMosaic.FluxFitParams(self.ffpSet[iexp]),
self.ccdSet[ichip], x0, y0)
metadata = measMosaic.metadataFromFluxFitParams(newP)
exp = afwImage.ExposureI(0,0)
exp.getMetadata().combine(metadata)
scale = self.fexp[iexp]*self.fchip[ichip]
constantPhotoCalib = afwImage.makePhotoCalibFromCalibZeroPoint(1.0/scale, 1.0/scale*std*M_LN10*0.4)
exp.setPhotoCalib(constantPhotoCalib)
try:
dataRef.put(exp, "fcr")
except Exception as e:
print("failed to write fcr: %s" % (e))
# Write the flux fit (including Jacobian) as a PhotoCalib for
# future compatibility with jointcal. This is redundant with
# the above, and should eventually supercede it.
detector = dataRef.get("camera")[dataRef.dataId["ccd"]]
nQuarter = detector.getOrientation().getNQuarter()
bbox = detector.getBBox()
try:
# Reading the Wcs we just wrote obviously isn't efficient, but
# it should be in the noise of the overall runtime and it
# saves us from doing a bunch of refactoring in a fragile
# package with no tests.
wcs = dataRef.get("jointcal_wcs")
except Exception as e:
print("failed to read Wcs for PhotoCalib: %s" % (e))
continue
bf = measMosaic.FluxFitBoundedField(bbox, newP, wcs,
zeroPoint=constantPhotoCalib.getInstFluxAtZeroMagnitude(),
nQuarter=nQuarter)
varyingPhotoCalib = afwImage.PhotoCalib(constantPhotoCalib.getCalibrationMean(),
constantPhotoCalib.getCalibrationErr(),
bf,
isConstant=False)
dataRef.put(varyingPhotoCalib, "jointcal_photoCalib")
def printMags(butler, dataId):
"""A simple function to print the RA and Dec, and magnitudes of sources
"""
# load the sources
sources = butler.get('src', dataId)
n = len(sources)
# get the fluxes as numpy arrays. For aperture fluxes, use getApFlux()
flux, ferr = sources.getPsfFlux(), sources.getPsfFluxErr()
mag, merr = 2.5 * numpy.log10(flux), 2.5 / numpy.log(10) * (ferr / flux)
# get the zeropoint, and apply ubercal correction if available
if butler.datasetExists('fcr_md', dataId):
fcr_md = butler.get("fcr_md", dataId)
ffp = measMosaic.FluxFitParams(fcr_md)
x, y = sources.getX(), sources.getY()
correction = numpy.array([ffp.eval(x[i], y[i]) for i in range(n)])
zeropoint = 2.5 * numpy.log10(fcr_md.get("FLUXMAG0")) + correction
else:
metadata = butler.get('calexp_md', dataId)
zeropoint = 2.5 * numpy.log10(metadata.get("FLUXMAG0"))
mag = zeropoint - mag
for i in range(n):
print sources[i].getRa().asDegrees(), sources[i].getDec().asDegrees(
), mag[i], merr[i]
def writeFcr(self, butler, ccdIdList, ccdSet, filterName, fexp, fchip,
ffpSet):
for ccdId in ccdIdList:
iexp, ichip = self.decodeCcdExposureId(ccdId)
if ichip not in ccdSet:
continue
x0 = 0.0
y0 = 0.0
newP = measMosaic.convertFluxFitParams(
measMosaic.FluxFitParams(ffpSet[iexp]), ccdSet[ichip], x0, y0)
metadata = measMosaic.metadataFromFluxFitParams(newP)
exp = afwImage.ExposureI(0, 0)
exp.getMetadata().combine(metadata)
scale = fexp[iexp] * fchip[ichip]
exp.setPhotoCalib(
afwImage.makePhotoCalibFromCalibZeroPoint(1.0 / scale))
exp.setFilter(afwImage.Filter(filterName))
try:
butler.put(exp, 'fcr', {'visit': iexp, 'ccd': ichip})
except Exception as e:
print("failed to write something: %s" % (e))
def run(self, matchLists, filterName, wcsList, butler):
if self.config.applyColorTerms:
ct = self.config.colorterms.selectColorTerm(filterName)
else:
ct = None
# Convert matchLists to meas_mosaic specific format
mlVisit = dict()
for ccdId in matchLists:
if matchLists[ccdId] is None:
continue
visit, ccd = self.decodeCcdExposureId(ccdId)
if visit not in mlVisit:
mlVisit[visit] = list()
matches = [m for m in matchLists[ccdId] if m[0] is not None]
keys = self.getKeys(matches[0][1].schema)
matches = self.selectMatches(matches, keys)
matches = self.selectStars(matches)
# Apply color term
if ct is not None and len(matches) != 0:
refSchema = matches[0][0].schema
key_p = refSchema.find(ct.primary).key
key_s = refSchema.find(ct.secondary).key
key_f = refSchema.find("flux").key
refFlux1 = numpy.array([m[0].get(key_p) for m in matches])
refFlux2 = numpy.array([m[0].get(key_s) for m in matches])
refMag1 = -2.5 * numpy.log10(refFlux1)
refMag2 = -2.5 * numpy.log10(refFlux2)
refMag = ct.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4 * refMag)
matches = [
self.setCatFlux(m, f, key_f)
for m, f in zip(matches, refFlux) if f == f
]
for m in matches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcsList[ccdId]),
measMosaic.Source(m[1]))
match[1].setExp(visit)
match[1].setChip(ccd)
mlVisit[visit].append(match)
matchList = []
for visit in mlVisit:
matchList.append(mlVisit[visit])
rootMat = measMosaic.kdtreeMat(matchList)
allMat = rootMat.mergeMat()
# Read CCD information
ccdSet = {}
for ccdId in matchLists:
if matchLists[ccdId] is None:
continue
visit, ccd = self.decodeCcdExposureId(ccdId)
if ccd not in ccdSet:
ccdDev = cameraGeomUtils.findCcd(butler.mapper.camera,
cameraGeom.Id(int(ccd)))
ccdSet[ccd] = ccdDev
# meas_mosaic specific wcs information
wcsDic = {}
for ccdId in wcsList:
visit, ccd = self.decodeCcdExposureId(ccdId)
if visit not in wcsDic and wcsList[ccdId] is not None:
wcs = wcsList[ccdId]
ccdDev = ccdSet[ccd]
offset = afwGeom.Extent2D(ccdDev.getCenter().getPixels(
ccdDev.getPixelSize()))
wcsDic[visit] = wcs.copyAtShiftedPixelOrigin(offset)
# meas_mosaic specific object list
matchVec = measMosaic.obsVecFromSourceGroup(allMat, wcsDic, ccdSet)
sourceVec = []
# Apply Jocabian correction calculated from wcs
for m in matchVec:
wcs = wcsList[m.iexp * 200 + m.ichip]
m.mag -= 2.5 * math.log10(
measMosaic.computeJacobian(wcs, afwGeom.Point2D(m.x, m.y)))
fluxFitOrder = self.config.fluxFitOrder
absolute = True
chebyshev = True
commonFluxCorr = False
solveCcdScale = True
ffpSet = {}
for visit in wcsDic:
ffp = measMosaic.FluxFitParams(fluxFitOrder, absolute, chebyshev)
u_max, v_max = self.getExtent(matchVec)
ffp.u_max = (math.floor(u_max / 10.) + 1) * 10
ffp.v_max = (math.floor(v_max / 10.) + 1) * 10
ffpSet[visit] = ffp
fexp = {}
fchip = {}
matchVec, sourceVec, wcsDic, ccdSet, fexp, fchip, ffpSet = measMosaic.fluxFit(
absolute, commonFluxCorr, matchVec, len(matchVec), sourceVec,
len(sourceVec), wcsDic, ccdSet, fexp, fchip, ffpSet, solveCcdScale)
self.writeFcr(butler, list(matchLists.keys()), ccdSet, filterName,
fexp, fchip, ffpSet)
return (1.0 / fexp[list(fexp.keys())[0]])
def mosaic(self,
dataRefList,
tractInfo,
ct=None,
debug=False,
diagDir=".",
diagnostics=False,
snapshots=False,
numCoresForReadSource=1,
readTimeout=9999,
verbose=False):
self.log.info(str(self.config))
self.outputDir = os.path.join(diagDir, "%04d" % tractInfo.getId())
if ((diagnostics or snapshots) and not os.path.isdir(self.outputDir)):
os.makedirs(self.outputDir)
if self.config.nBrightest != 0:
self.log.fatal("Config paremeter nBrightest is deprecated.")
self.log.fatal("Please use cellSize and nStarPerCell.")
self.log.fatal("Exiting ...")
return []
dataRefListOverlapWithTract, dataRefListToUse = self.checkOverlapWithTract(
tractInfo, dataRefList)
sourceSet, matchList, dataRefListUsed = self.readCatalog(
dataRefListToUse, ct, numCoresForReadSource, readTimeout, verbose)
dataRefListToOutput = list(
set(dataRefListUsed) & set(dataRefListOverlapWithTract))
ccdSet = self.readCcd(dataRefListUsed)
if debug:
for ccd in ccdSet.values():
self.log.info(
str(ccd.getId().getSerial()) + " " +
str(ccd.getCenter().getPixels(ccd.getPixelSize())) + " " +
str(ccd.getOrientation().getYaw()))
wcsDic = self.readWcs(dataRefListUsed, ccdSet)
self.removeNonExistCcd(dataRefListUsed, ccdSet)
if debug:
for iexp, wcs in wcsDic.items():
self.log.info(
str(iexp) + " " + str(wcs.getPixelOrigin()) + " " +
str(wcs.getSkyOrigin().getPosition(afwGeom.degrees)))
self.log.info("frameIds : " + str(list(wcsDic.keys())))
self.log.info("ccdIds : " + str(list(ccdSet.keys())))
d_lim = afwGeom.Angle(self.config.radXMatch, afwGeom.arcseconds)
if debug:
self.log.info("d_lim : %f" % d_lim)
allMat, allSource = self.mergeCatalog(sourceSet, matchList, ccdSet,
d_lim)
self.log.info("Flag suspect objects")
measMosaic.flagSuspect(allMat, allSource, wcsDic)
if self.config.clipSourcesOutsideTract:
tractBBox = afwGeom.Box2D(tractInfo.getBBox())
tractWcs = tractInfo.getWcs()
allSourceClipped = [
ss for ss in allSource
if tractBBox.contains(tractWcs.skyToPixel(ss[0].getSky()))
]
self.log.info("Num of allSources: %d" % (len(allSource)))
self.log.info("Num of clipped allSources: %d" %
(len(allSourceClipped)))
allSource = allSourceClipped
self.log.info("Make obsVec")
nmatch = len(allMat)
nsource = len(allSource)
matchVec = measMosaic.obsVecFromSourceGroup(allMat, wcsDic, ccdSet)
sourceVec = measMosaic.obsVecFromSourceGroup(allSource, wcsDic, ccdSet)
self.log.info("Solve mosaic ...")
order = self.config.fittingOrder
internal = self.config.internalFitting
solveCcd = self.config.solveCcd
allowRotation = self.config.allowRotation
fluxFitOrder = self.config.fluxFitOrder
chebyshev = self.config.chebyshev
absolute = self.config.fluxFitAbsolute
solveCcdScale = self.config.fluxFitSolveCcd
catRMS = self.config.catRMS
if not internal:
sourceVec = []
if debug:
self.log.info("order : %d" % order)
self.log.info("internal : %r" % internal)
self.log.info("solveCcd : %r " % solveCcd)
self.log.info("allowRotation : %r" % allowRotation)
if self.config.doSolveWcs:
if internal:
coeffSet, matchVec, sourceVec, wcsDic, ccdSet = measMosaic.solveMosaic_CCD(
order, nmatch, nsource, matchVec, sourceVec, wcsDic,
ccdSet, solveCcd, allowRotation, verbose, catRMS,
snapshots, self.outputDir)
else:
coeffSet, matchVec, wcsDic, ccdSet = measMosaic.solveMosaic_CCD_shot(
order, nmatch, matchVec, wcsDic, ccdSet, solveCcd,
allowRotation, verbose, catRMS, snapshots, self.outputDir)
self.matchVec = matchVec
self.sourceVec = sourceVec
self.wcsDic = wcsDic
self.ccdSet = ccdSet
self.coeffSet = coeffSet
self.writeNewWcs(dataRefListToOutput)
if diagnostics:
self.outputDiagWcs()
for m in matchVec:
coeff = coeffSet[m.iexp]
scale = coeff.pixelScale()
m.mag -= 2.5 * math.log10(coeff.detJ(m.u, m.v) / scale**2)
if len(sourceVec) != 0:
for s in sourceVec:
coeff = coeffSet[s.iexp]
scale = coeff.pixelScale()
s.mag -= 2.5 * math.log10(coeff.detJ(s.u, s.v) / scale**2)
else:
wcsAll = dict()
for dataRef in dataRefListUsed:
frameId = "%07d-%03d" % (dataRef.dataId["visit"],
dataRef.dataId["ccd"])
md = dataRef.get("calexp_md")
wcsAll[frameId] = afwGeom.makeSkyWcs(md)
del md
for m in matchVec:
wcs = wcsAll["%07d-%03d" % (m.iexp, m.ichip)]
m.mag -= 2.5 * math.log10(
measMosaic.computeJacobian(wcs, afwGeom.Point2D(m.x, m.y)))
if len(sourceVec) != 0:
for s in sourceVec:
wcs = wcsAll["%07d-%03d" % (s.iexp, s.ichip)]
s.mag -= 2.5 * math.log10(
measMosaic.computeJacobian(wcs,
afwGeom.Point2D(s.x, s.y)))
del wcsAll
if self.config.doSolveFlux:
ffpSet = {}
for visit in wcsDic:
ffp = measMosaic.FluxFitParams(fluxFitOrder, absolute,
chebyshev)
u_max, v_max = mosaicUtils.getExtent(matchVec)
ffp.u_max = (math.floor(u_max / 10.0) + 1) * 10
ffp.v_max = (math.floor(v_max / 10.0) + 1) * 10
ffpSet[visit] = ffp
fexp = {}
fchip = {}
matchVec, sourceVec, wcsDic, ccdSet, fexp, fchip, ffpSet = measMosaic.fluxFit(
absolute, self.config.commonFluxCorr, matchVec, len(matchVec),
sourceVec, len(sourceVec), wcsDic, ccdSet, fexp, fchip, ffpSet,
solveCcdScale)
self.ffpSet = ffpSet
self.fexp = fexp
self.fchip = fchip
self.writeFcr(dataRefListToOutput)
if diagnostics:
self.outputDiagFlux()
if diagnostics and self.config.doSolveWcs and self.config.doSolveFlux:
if len(sourceVec) != 0:
mosaicUtils.writeCatalog(
coeffSet, ffpSet, fexp, fchip, matchVec, sourceVec,
os.path.join(self.outputDir, "catalog.fits"))
return list(wcsDic.keys())
def run(self, dataRefList, ct=None, debug=False, verbose=False):
ccdSet = self.readCcd(dataRefList)
self.removeNonExistCcd(dataRefList, ccdSet)
sourceSet = []
matchList = []
astrom = measAstrom.ANetBasicAstrometryTask(self.config.astrom)
ssVisit = dict()
mlVisit = dict()
dataRefListUsed = list()
wcsDic = dict()
calibDic = dict()
ffpDic = dict()
for dataRef in dataRefList:
if dataRef.dataId['visit'] not in ssVisit:
ssVisit[dataRef.dataId['visit']] = list()
mlVisit[dataRef.dataId['visit']] = list()
wcsDic[dataRef.dataId['visit']] = dict()
calibDic[dataRef.dataId['visit']] = dict()
ffpDic[dataRef.dataId['visit']] = dict()
try:
if not dataRef.datasetExists('src'):
raise RuntimeError("no data for src %s" % (dataRef.dataId))
if not dataRef.datasetExists('jointcal_wcs'):
raise RuntimeError("no data for wcs %s" % (dataRef.dataId))
if not dataRef.datasetExists('fcr'):
raise RuntimeError("no data for fcr %s" % (dataRef.dataId))
wcs = dataRef.get('jointcal_wcs')
md = dataRef.get('calexp_md')
filterName = afwImage.Filter(md).getName()
md = dataRef.get('fcr_md')
ffp = measMosaic.FluxFitParams(md)
photoCalib = afwImage.makePhotoCalibFromMetadata(md)
sources = dataRef.get('src',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
icSrces = dataRef.get('icSrc',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
packedMatches = dataRef.get('icMatch')
matches = astrom.joinMatchListWithCatalog(packedMatches, icSrces)
matches = [m for m in matches if m[0] is not None]
if matches:
refSchema = matches[0][0].schema
if ct:
# Add a "flux" field to the match records which contains the
# colorterm-corrected reference flux. The field name is hard-coded in
# lsst::meas::mosaic::Source.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
key_f = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
key_p = refSchema.find(refSchema.join(ct.primary, "flux")).key
key_s = refSchema.find(refSchema.join(ct.secondary, "flux")).key
refFlux1 = numpy.array([m[0].get(key_p) for m in matches])
refFlux2 = numpy.array([m[0].get(key_s) for m in matches])
refMag1 = -2.5*numpy.log10(refFlux1)
refMag2 = -2.5*numpy.log10(refFlux2)
refMag = ct.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4*refMag)
matches = [setCatFlux(m, f, key_f) for m, f in zip(matches, refFlux) if f == f]
else:
# No colorterm; we can get away with aliasing the reference flux.
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
sources = self.selectStars(sources)
matches = self.selectStars(matches, True)
except Exception as e:
print("Failed to read: %s" % (e))
sources = None
continue
if sources is not None:
for s in sources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataRef.dataId['visit'])
src.setChip(dataRef.dataId['ccd'])
ssVisit[dataRef.dataId['visit']].append(src)
for m in matches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1]))
match[1].setExp(dataRef.dataId['visit'])
match[1].setChip(dataRef.dataId['ccd'])
mlVisit[dataRef.dataId['visit']].append(match)
wcsDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = wcs
calibDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = photoCalib
ffpDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = ffp
dataRefListUsed.append(dataRef)
for visit in ssVisit:
sourceSet.append(ssVisit[visit])
matchList.append(mlVisit[visit])
d_lim = afwGeom.Angle(self.config.radXMatch, afwGeom.arcseconds)
nbrightest = self.config.nBrightest
allMat, allSource = self.mergeCatalog(sourceSet, matchList, ccdSet, d_lim)
dx_m, dy_m, dx_s, dy_s, m0_m, dm_m, m0_s, dm_s = self.makeDiffPosFlux(allMat, allSource, wcsDic, calibDic, ffpDic)
self.plotFlux(m0_m, dm_m, m0_s, dm_s)
self.makeFluxStat(allMat, allSource, calibDic, ffpDic, wcsDic)
self.plotPos(dx_m, dy_m, dx_s, dy_s)
self.plotPosAsMag(m0_s, dx_s, dy_s)
self.writeCatalog(allSource, wcsDic, calibDic, ffpDic)