def readSrc(self, dataRef):
"""Read source catalog etc for input dataRef
The following are returned:
Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md',
respectively.
NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane
coordinate system), the (x, y) pixel values of the centroid slot for the source
catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system
expected by meas_mosaic).
If color transformation information is given, it will be applied to the reference flux
of the matched list. The source catalog and matched list will be converted to measMosaic's
Source and SourceMatch and returned.
The number of 'Source's in each cell defined by config.cellSize will be limited to brightest
config.nStarPerCell.
"""
self.log = Log.getDefaultLogger()
dataId = dataRef.dataId
try:
if not dataRef.datasetExists("src"):
raise RuntimeError("no data for src %s" % (dataId))
if not dataRef.datasetExists("calexp_md"):
raise RuntimeError("no data for calexp_md %s" % (dataId))
calexp_md = dataRef.get("calexp_md", immediate=True)
detector = dataRef.get("camera")[dataRef.dataId[
"ccd"]] # OK for HSC; maybe not for other cameras
wcs = afwGeom.makeSkyWcs(calexp_md)
nQuarter = detector.getOrientation().getNQuarter()
sources = dataRef.get("src",
immediate=True,
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
# Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is
# required, but alias mapping must be set to associate HSC's schema with that of LSST.
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is None:
if nQuarter % 4 != 0:
dims = afwImage.bboxFromMetadata(calexp_md).getDimensions()
sources = mosaicUtils.rotatePixelCoords(
sources, dims.getX(), dims.getY(), nQuarter)
# Set the aliap map for the source catalog
if self.config.srcSchemaMap is not None and hscRun is not None:
aliasMap = sources.schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items():
aliasMap.set(lsstName, otherName)
refObjLoader = self.config.loadAstrom.apply(
butler=dataRef.getButler())
srcMatch = dataRef.get("srcMatch", immediate=True)
if hscRun is not None:
# The reference object loader grows the bbox by the config parameter pixelMargin. This
# is set to 50 by default but is not reflected by the radius parameter set in the
# metadata, so some matches may reside outside the circle searched within this radius
# Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to
# accommodate.
matchmeta = srcMatch.table.getMetadata()
rad = matchmeta.getDouble("RADIUS")
matchmeta.setDouble(
"RADIUS", rad * 1.05,
"field radius in degrees, approximate, padded")
matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources)
# Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match)
if self.config.srcSchemaMap is not None and hscRun is not None:
for mm in matches:
aliasMap = mm[1].schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items(
):
aliasMap.set(lsstName, otherName)
if hscRun is not None:
for slot in ("PsfFlux", "ModelFlux", "ApFlux", "InstFlux",
"Centroid", "Shape"):
getattr(matches[0][1].getTable(), "define" + slot)(getattr(
sources, "get" + slot + "Definition")())
# For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so
# set it to flux_naive explicitly
for slot in ("CalibFlux", ):
getattr(matches[0][1].getTable(),
"define" + slot)("flux_naive")
matches = [m for m in matches if m[0] is not None]
refSchema = matches[0][0].schema if matches else None
if self.cterm is not None and len(matches) != 0:
# Add a "flux" field to the input schema of the first element
# of the match and populate it with a colorterm correct flux.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
fluxKey = mapper.editOutputSchema().addField(
"flux", type=float, doc="Reference flux")
fluxSigmaKey = mapper.editOutputSchema().addField(
"fluxSigma", type=float, doc="Reference flux uncertainty")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
primaryFluxKey = refSchema.find(
refSchema.join(self.cterm.primary, "flux")).key
secondaryFluxKey = refSchema.find(
refSchema.join(self.cterm.secondary, "flux")).key
primaryFluxSigmaKey = refSchema.find(
refSchema.join(self.cterm.primary, "fluxSigma")).key
secondaryFluxSigmaKey = refSchema.find(
refSchema.join(self.cterm.secondary, "fluxSigma")).key
refFlux1 = numpy.array(
[m[0].get(primaryFluxKey) for m in matches])
refFlux2 = numpy.array(
[m[0].get(secondaryFluxKey) for m in matches])
refFluxSigma1 = numpy.array(
[m[0].get(primaryFluxSigmaKey) for m in matches])
refFluxSigma2 = numpy.array(
[m[0].get(secondaryFluxSigmaKey) for m in matches])
refMag1 = -2.5 * numpy.log10(refFlux1)
refMag2 = -2.5 * numpy.log10(refFlux2)
refMag = self.cterm.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4 * refMag)
refFluxSigma = self.cterm.propagateFluxErrors(
refFluxSigma1, refFluxSigma2)
matches = [
self.setCatFlux(m, flux, fluxKey, fluxSigma, fluxSigmaKey)
for m, flux, fluxSigma in zip(matches, refFlux,
refFluxSigma) if flux == flux
]
else:
filterName = afwImage.Filter(calexp_md).getName()
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
# LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN.
matches = mosaicUtils.matchJanskyToDn(matches)
selSources = self.selectStars(sources,
self.config.includeSaturated)
selMatches = self.selectStars(matches,
self.config.includeSaturated)
retSrc = list()
retMatch = list()
if len(selMatches) > self.config.minNumMatch:
naxis1, naxis2 = afwImage.bboxFromMetadata(
calexp_md).getDimensions()
if hscRun is None:
if nQuarter % 2 != 0:
naxis1, naxis2 = naxis2, naxis1
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(naxis1, naxis2))
cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize,
self.config.cellSize)
for s in selSources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataId["visit"])
src.setChip(dataId["ccd"])
try:
tmp = measMosaic.SpatialCellSource(src)
cellSet.insertCandidate(tmp)
except:
self.log.info(
"FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f"
% (dataRef.dataId["visit"], dataRef.
dataId["ccd"], src.getX(), src.getY()) +
" bbox=" + str(bbox))
for cell in cellSet.getCellList():
cell.sortCandidates()
for i, cand in enumerate(cell):
src = cand.getSource()
retSrc.append(src)
if i == self.config.nStarPerCell - 1:
break
for m in selMatches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs),
measMosaic.Source(m[1]))
match[1].setExp(dataId["visit"])
match[1].setChip(dataId["ccd"])
retMatch.append(match)
else:
self.log.info(
"%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD"
% (dataRef.dataId["visit"], dataRef.dataId["ccd"],
len(selMatches), len(matches)))
except Exception as e:
self.log.warn("Failed to read %s: %s" % (dataId, e))
return dataId, [None, None, None]
return dataId, [retSrc, retMatch, wcs]
def run(self, matchLists, filterName, wcsList, butler):
if self.config.applyColorTerms:
ct = self.config.colorterms.selectColorTerm(filterName)
else:
ct = None
# Convert matchLists to meas_mosaic specific format
mlVisit = dict()
for ccdId in matchLists:
if matchLists[ccdId] is None:
continue
visit, ccd = self.decodeCcdExposureId(ccdId)
if visit not in mlVisit:
mlVisit[visit] = list()
matches = [m for m in matchLists[ccdId] if m[0] is not None]
keys = self.getKeys(matches[0][1].schema)
matches = self.selectMatches(matches, keys)
matches = self.selectStars(matches)
# Apply color term
if ct is not None and len(matches) != 0:
refSchema = matches[0][0].schema
key_p = refSchema.find(ct.primary).key
key_s = refSchema.find(ct.secondary).key
key_f = refSchema.find("flux").key
refFlux1 = numpy.array([m[0].get(key_p) for m in matches])
refFlux2 = numpy.array([m[0].get(key_s) for m in matches])
refMag1 = -2.5 * numpy.log10(refFlux1)
refMag2 = -2.5 * numpy.log10(refFlux2)
refMag = ct.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4 * refMag)
matches = [
self.setCatFlux(m, f, key_f)
for m, f in zip(matches, refFlux) if f == f
]
for m in matches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcsList[ccdId]),
measMosaic.Source(m[1]))
match[1].setExp(visit)
match[1].setChip(ccd)
mlVisit[visit].append(match)
matchList = []
for visit in mlVisit:
matchList.append(mlVisit[visit])
rootMat = measMosaic.kdtreeMat(matchList)
allMat = rootMat.mergeMat()
# Read CCD information
ccdSet = {}
for ccdId in matchLists:
if matchLists[ccdId] is None:
continue
visit, ccd = self.decodeCcdExposureId(ccdId)
if ccd not in ccdSet:
ccdDev = cameraGeomUtils.findCcd(butler.mapper.camera,
cameraGeom.Id(int(ccd)))
ccdSet[ccd] = ccdDev
# meas_mosaic specific wcs information
wcsDic = {}
for ccdId in wcsList:
visit, ccd = self.decodeCcdExposureId(ccdId)
if visit not in wcsDic and wcsList[ccdId] is not None:
wcs = wcsList[ccdId]
ccdDev = ccdSet[ccd]
offset = afwGeom.Extent2D(ccdDev.getCenter().getPixels(
ccdDev.getPixelSize()))
wcsDic[visit] = wcs.copyAtShiftedPixelOrigin(offset)
# meas_mosaic specific object list
matchVec = measMosaic.obsVecFromSourceGroup(allMat, wcsDic, ccdSet)
sourceVec = []
# Apply Jocabian correction calculated from wcs
for m in matchVec:
wcs = wcsList[m.iexp * 200 + m.ichip]
m.mag -= 2.5 * math.log10(
measMosaic.computeJacobian(wcs, afwGeom.Point2D(m.x, m.y)))
fluxFitOrder = self.config.fluxFitOrder
absolute = True
chebyshev = True
commonFluxCorr = False
solveCcdScale = True
ffpSet = {}
for visit in wcsDic:
ffp = measMosaic.FluxFitParams(fluxFitOrder, absolute, chebyshev)
u_max, v_max = self.getExtent(matchVec)
ffp.u_max = (math.floor(u_max / 10.) + 1) * 10
ffp.v_max = (math.floor(v_max / 10.) + 1) * 10
ffpSet[visit] = ffp
fexp = {}
fchip = {}
matchVec, sourceVec, wcsDic, ccdSet, fexp, fchip, ffpSet = measMosaic.fluxFit(
absolute, commonFluxCorr, matchVec, len(matchVec), sourceVec,
len(sourceVec), wcsDic, ccdSet, fexp, fchip, ffpSet, solveCcdScale)
self.writeFcr(butler, list(matchLists.keys()), ccdSet, filterName,
fexp, fchip, ffpSet)
return (1.0 / fexp[list(fexp.keys())[0]])
def run(self, dataRefList, ct=None, debug=False, verbose=False):
ccdSet = self.readCcd(dataRefList)
self.removeNonExistCcd(dataRefList, ccdSet)
sourceSet = []
matchList = []
astrom = measAstrom.ANetBasicAstrometryTask(self.config.astrom)
ssVisit = dict()
mlVisit = dict()
dataRefListUsed = list()
wcsDic = dict()
calibDic = dict()
ffpDic = dict()
for dataRef in dataRefList:
if dataRef.dataId['visit'] not in ssVisit:
ssVisit[dataRef.dataId['visit']] = list()
mlVisit[dataRef.dataId['visit']] = list()
wcsDic[dataRef.dataId['visit']] = dict()
calibDic[dataRef.dataId['visit']] = dict()
ffpDic[dataRef.dataId['visit']] = dict()
try:
if not dataRef.datasetExists('src'):
raise RuntimeError("no data for src %s" % (dataRef.dataId))
if not dataRef.datasetExists('jointcal_wcs'):
raise RuntimeError("no data for wcs %s" % (dataRef.dataId))
if not dataRef.datasetExists('fcr'):
raise RuntimeError("no data for fcr %s" % (dataRef.dataId))
wcs = dataRef.get('jointcal_wcs')
md = dataRef.get('calexp_md')
filterName = afwImage.Filter(md).getName()
md = dataRef.get('fcr_md')
ffp = measMosaic.FluxFitParams(md)
photoCalib = afwImage.makePhotoCalibFromMetadata(md)
sources = dataRef.get('src',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
icSrces = dataRef.get('icSrc',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS,
immediate=True)
packedMatches = dataRef.get('icMatch')
matches = astrom.joinMatchListWithCatalog(packedMatches, icSrces)
matches = [m for m in matches if m[0] is not None]
if matches:
refSchema = matches[0][0].schema
if ct:
# Add a "flux" field to the match records which contains the
# colorterm-corrected reference flux. The field name is hard-coded in
# lsst::meas::mosaic::Source.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
key_f = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
key_p = refSchema.find(refSchema.join(ct.primary, "flux")).key
key_s = refSchema.find(refSchema.join(ct.secondary, "flux")).key
refFlux1 = numpy.array([m[0].get(key_p) for m in matches])
refFlux2 = numpy.array([m[0].get(key_s) for m in matches])
refMag1 = -2.5*numpy.log10(refFlux1)
refMag2 = -2.5*numpy.log10(refFlux2)
refMag = ct.transformMags(refMag1, refMag2)
refFlux = numpy.power(10.0, -0.4*refMag)
matches = [setCatFlux(m, f, key_f) for m, f in zip(matches, refFlux) if f == f]
else:
# No colorterm; we can get away with aliasing the reference flux.
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
sources = self.selectStars(sources)
matches = self.selectStars(matches, True)
except Exception as e:
print("Failed to read: %s" % (e))
sources = None
continue
if sources is not None:
for s in sources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataRef.dataId['visit'])
src.setChip(dataRef.dataId['ccd'])
ssVisit[dataRef.dataId['visit']].append(src)
for m in matches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1]))
match[1].setExp(dataRef.dataId['visit'])
match[1].setChip(dataRef.dataId['ccd'])
mlVisit[dataRef.dataId['visit']].append(match)
wcsDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = wcs
calibDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = photoCalib
ffpDic[dataRef.dataId['visit']][dataRef.dataId['ccd']] = ffp
dataRefListUsed.append(dataRef)
for visit in ssVisit:
sourceSet.append(ssVisit[visit])
matchList.append(mlVisit[visit])
d_lim = afwGeom.Angle(self.config.radXMatch, afwGeom.arcseconds)
nbrightest = self.config.nBrightest
allMat, allSource = self.mergeCatalog(sourceSet, matchList, ccdSet, d_lim)
dx_m, dy_m, dx_s, dy_s, m0_m, dm_m, m0_s, dm_s = self.makeDiffPosFlux(allMat, allSource, wcsDic, calibDic, ffpDic)
self.plotFlux(m0_m, dm_m, m0_s, dm_s)
self.makeFluxStat(allMat, allSource, calibDic, ffpDic, wcsDic)
self.plotPos(dx_m, dy_m, dx_s, dy_s)
self.plotPosAsMag(m0_s, dx_s, dy_s)
self.writeCatalog(allSource, wcsDic, calibDic, ffpDic)