def make_source_catalog_from_astropy_table(out_table, debug=False):
"""Return an AFW SourceCatalog from an Astropy Table
Written with extensive reference to
https://github.com/lsst/meas_astrom/blob/master/convertToFitsTable.py
"""
filters = ('J', 'H', 'K')
schema = makeMinimalSchema(filters=filters, debug=debug)
out_cat = afwTable.SourceCatalog(schema)
for row in out_table:
record = out_cat.addNew()
record.setId(twomass_int_id(row['2MASSID']))
record.setRa(float(row['coord_ra']) * lsst.afw.geom.degrees)
record.setDec(float(row['coord_dec']) * lsst.afw.geom.degrees)
for filt in filters:
filtMag = '%s_mag' % filt
filtMagSigma = '%s_mag_sigma' % filt
filtFlux = '%s_flux' % filt
filtFluxSigma = '%s_fluxSigma' % filt
abMag = vegaToABMag(row[filtMag], filt) # error remains unchanged
record.set(filtFlux, fluxFromABMag(abMag))
record.set(filtFluxSigma, fluxErrFromABMagErr(row[filtMagSigma], abMag))
return out_cat
def _getFluxes(self, inputData):
"""Compute the flux fields that will go into the output catalog.
Parameters
----------
inputData : `numpy.ndarray`
The input data to compute fluxes for.
Returns
-------
fluxes : `dict` [`str`, `numpy.ndarray`]
The values that will go into the flux and fluxErr fields in the
output catalog.
"""
result = {}
for item in self.config.mag_column_list:
result[item + '_flux'] = (inputData[item] * u.ABmag).to_value(
u.nJy)
if len(self.config.mag_err_column_map) > 0:
for err_key in self.config.mag_err_column_map.keys():
error_col_name = self.config.mag_err_column_map[err_key]
# TODO: multiply by 1e9 here until we have a replacement (see DM-16903)
# NOTE: copy the arrays because the numpy strides may not be useable by C++.
fluxErr = fluxErrFromABMagErr(inputData[error_col_name].copy(),
inputData[err_key].copy()) * 1e9
result[err_key + '_fluxErr'] = fluxErr
return result
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
applyColorterms=False):
"""Load the necessary reference catalog sources, convert fluxes to
correct units, and apply color term corrections if requested.
Parameters
----------
refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
The reference catalog loader to use to get the data.
referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
Source selector to apply to loaded reference catalog.
center : `lsst.geom.SpherePoint`
The center around which to load sources.
radius : `lsst.geom.Angle`
The radius around ``center`` to load sources in.
filterName : `str`
The name of the camera filter to load fluxes for.
applyColorterms : `bool`
Apply colorterm corrections to the refcat for ``filterName``?
Returns
-------
refCat : `lsst.afw.table.SimpleCatalog`
The loaded reference catalog.
fluxField : `str`
The name of the reference catalog flux field appropriate for ``filterName``.
"""
skyCircle = refObjLoader.loadSkyCircle(center,
afwGeom.Angle(radius, afwGeom.radians),
filterName)
selected = referenceSelector.run(skyCircle.refCat)
# Need memory contiguity to get reference filters as a vector.
if not selected.sourceCat.isContiguous():
refCat = selected.sourceCat.copy(deep=True)
else:
refCat = selected.sourceCat
if applyColorterms:
try:
refCatName = refObjLoader.ref_dataset_name
except AttributeError:
# NOTE: we need this try:except: block in place until we've completely removed a.net support.
raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.")
self.log.info("Applying color terms for filterName=%r reference catalog=%s",
filterName, refCatName)
colorterm = self.config.colorterms.getColorterm(
filterName=filterName, photoCatName=refCatName, doRaise=True)
refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
# TODO: I didn't want to use this, but I'll deal with it in DM-16903
refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
return refCat, skyCircle.fluxField
def _set_mags(self, record, row, key_map):
"""!Set the flux records from the input magnitudes
@param[in,out] record SourceCatalog record to modify
@param[in] row dict like object containing magnitude values
@param[in] key_map Map of catalog keys to use in filling the record
"""
for item in self.config.mag_column_list:
record.set(key_map[item + "_flux"], fluxFromABMag(row[item]))
if len(self.config.mag_err_column_map) > 0:
for err_key in self.config.mag_err_column_map.keys():
error_col_name = self.config.mag_err_column_map[err_key]
record.set(key_map[err_key + "_fluxSigma"], fluxErrFromABMagErr(row[error_col_name], row[err_key]))
def testBasics(self):
for flux in (1, 210, 3210, 43210, 543210):
abMag = afwImage.abMagFromFlux(flux)
self.assertAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertAlmostEqual(flux, fluxRoundTrip)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertAlmostEqual(fluxErr, fluxErrRoundTrip)
def _set_mags(self, record, row, key_map):
"""!Set the flux records from the input magnitudes
@param[in,out] record SourceCatalog record to modify
@param[in] row dict like object containing magnitude values
@param[in] key_map Map of catalog keys to use in filling the record
"""
for item in self.config.mag_column_list:
record.set(key_map[item + '_flux'], fluxFromABMag(row[item]))
if len(self.config.mag_err_column_map) > 0:
for err_key in self.config.mag_err_column_map.keys():
error_col_name = self.config.mag_err_column_map[err_key]
record.set(
key_map[err_key + '_fluxSigma'],
fluxErrFromABMagErr(row[error_col_name], row[err_key]))
def testVector(self):
flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0])
flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0>
abMag = afwImage.abMagFromFlux(flux)
self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertFloatsAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertFloatsAlmostEqual(fluxErr, fluxErrRoundTrip, rtol=1.0e-15)
def testVector(self):
flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0])
flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0>
abMag = afwImage.abMagFromFlux(flux)
self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertFloatsAlmostEqual(abMagErr,
refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertFloatsAlmostEqual(fluxErr,
fluxErrRoundTrip,
rtol=1.0e-15)
def _formatCatalog(self, fgcmStarCat, offsets):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat: `afwTable.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets: `list` with len(self.bands) entries
Zeropoint offsets to apply
Returns
-------
formattedCat: `afwTable.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
self.bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in self.bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(self.bands):
mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b]
# We want fluxes in Jy from calibrated AB magnitudes
# (after applying offset)
# TODO: Full implementation of RFC-549 will have all reference
# catalogs in nJy instead of Jy.
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(
fgcmStarCat['magErr_std'][:, b], mag)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
return formattedCat
def _setFlux(self, record, row, key_map):
"""Set flux fields in a record of an indexed catalog.
Parameters
----------
record : `lsst.afw.table.SimpleRecord`
Row from indexed catalog to modify.
row : structured `numpy.array`
Row from catalog being ingested.
key_map : `dict` mapping `str` to `lsst.afw.table.Key`
Map of catalog keys.
"""
for item in self.config.mag_column_list:
record.set(key_map[item + '_flux'], fluxFromABMag(row[item]))
if len(self.config.mag_err_column_map) > 0:
for err_key in self.config.mag_err_column_map.keys():
error_col_name = self.config.mag_err_column_map[err_key]
record.set(
key_map[err_key + '_fluxErr'],
fluxErrFromABMagErr(row[error_col_name], row[err_key]))
def _formatCatalog(self, fgcmStarCat, offsets):
"""
Turn an FGCM-formatted star catalog, applying zp offsets.
parameters
----------
fgcmStarCat: SimpleCatalog
SimpleCatalog as output by fgcmcal
offsets: list with len(self.bands) entries
Zeropoint offsets to apply
returns
-------
formattedCat: SimpleCatalog
SimpleCatalog suitable for ref_cat
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
for band in self.bands:
sourceMapper.editOutputSchema().addField('%s_flux' % (band),
type=np.float64)
sourceMapper.editOutputSchema().addField('%s_fluxErr' % (band),
type=np.float64)
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.float64)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
for b, band in enumerate(self.bands):
mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b]
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(
fgcmStarCat['magerr_std'][:, b], mag)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
return formattedCat
def _setFlux(self, record, row, key_map):
"""Set flux fields in a record of an indexed catalog.
Parameters
----------
record : `lsst.afw.table.SimpleRecord`
Row from indexed catalog to modify.
row : structured `numpy.array`
Row from catalog being ingested.
key_map : `dict` mapping `str` to `lsst.afw.table.Key`
Map of catalog keys.
"""
for item in self.config.mag_column_list:
record.set(key_map[item+'_flux'], (row[item]*u.ABmag).to_value(u.nJy))
if len(self.config.mag_err_column_map) > 0:
for err_key in self.config.mag_err_column_map.keys():
error_col_name = self.config.mag_err_column_map[err_key]
# TODO: multiply by 1e9 here until we have a replacement (see DM-16903)
fluxErr = fluxErrFromABMagErr(row[error_col_name], row[err_key])
if fluxErr is not None:
fluxErr *= 1e9
record.set(key_map[err_key+'_fluxErr'], fluxErr)
def readSrc(self, dataRef):
"""Read source catalog etc for input dataRef
The following are returned:
Source catalog, matched list, and wcs will be read from 'src', 'srcMatch', and 'calexp_md',
respectively.
NOTE: If the detector has nQuarter%4 != 0 (i.e. it is rotated w.r.t the focal plane
coordinate system), the (x, y) pixel values of the centroid slot for the source
catalogs are rotated such that pixel (0, 0) is the LLC (i.e. the coordinate system
expected by meas_mosaic).
If color transformation information is given, it will be applied to the reference flux
of the matched list. The source catalog and matched list will be converted to measMosaic's
Source and SourceMatch and returned.
The number of 'Source's in each cell defined by config.cellSize will be limited to brightest
config.nStarPerCell.
"""
self.log = Log.getDefaultLogger()
dataId = dataRef.dataId
try:
if not dataRef.datasetExists("src"):
raise RuntimeError("no data for src %s" % (dataId))
if not dataRef.datasetExists("calexp_md"):
raise RuntimeError("no data for calexp_md %s" % (dataId))
calexp_md = dataRef.get("calexp_md", immediate=True)
detector = dataRef.get("camera")[dataRef.dataId["ccd"]] # OK for HSC; maybe not for other cameras
wcs = afwGeom.makeSkyWcs(calexp_md)
nQuarter = detector.getOrientation().getNQuarter()
sources = dataRef.get("src", immediate=True, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
# Check if we are looking at HSC stack outputs: if so, no pixel rotation of sources is
# required, but alias mapping must be set to associate HSC's schema with that of LSST.
hscRun = mosaicUtils.checkHscStack(calexp_md)
if hscRun is None:
if nQuarter%4 != 0:
dims = afwImage.bboxFromMetadata(calexp_md).getDimensions()
sources = mosaicUtils.rotatePixelCoords(sources, dims.getX(), dims.getY(),
nQuarter)
# Set some alias maps for the source catalog where needed for
# backwards compatibility
if self.config.srcSchemaMap and hscRun:
aliasMap = sources.schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items():
aliasMap.set(lsstName, otherName)
if self.config.flagsToAlias and "calib_psfUsed" in sources.schema:
aliasMap = sources.schema.getAliasMap()
for lsstName, otherName in self.config.flagsToAlias.items():
aliasMap.set(lsstName, otherName)
refObjLoader = self.config.loadAstrom.apply(butler=dataRef.getButler())
srcMatch = dataRef.get("srcMatch", immediate=True)
if hscRun is not None:
# The reference object loader grows the bbox by the config parameter pixelMargin. This
# is set to 50 by default but is not reflected by the radius parameter set in the
# metadata, so some matches may reside outside the circle searched within this radius
# Thus, increase the radius set in the metadata fed into joinMatchListWithCatalog() to
# accommodate.
matchmeta = srcMatch.table.getMetadata()
rad = matchmeta.getDouble("RADIUS")
matchmeta.setDouble("RADIUS", rad*1.05, "field radius in degrees, approximate, padded")
matches = refObjLoader.joinMatchListWithCatalog(srcMatch, sources)
# Set the aliap map for the matched sources (i.e. the [1] attribute schema for each match)
if self.config.srcSchemaMap is not None and hscRun is not None:
for mm in matches:
aliasMap = mm[1].schema.getAliasMap()
for lsstName, otherName in self.config.srcSchemaMap.items():
aliasMap.set(lsstName, otherName)
if hscRun is not None:
for slot in ("PsfFlux", "ModelFlux", "ApFlux", "GaussianFlux", "Centroid", "Shape"):
getattr(matches[0][1].getTable(), "define" + slot)(
getattr(sources, "get" + slot + "Definition")())
# For some reason, the CalibFlux slot in sources is coming up as centroid_sdss, so
# set it to flux_naive explicitly
for slot in ("CalibFlux", ):
getattr(matches[0][1].getTable(), "define" + slot)("flux_naive")
matches = [m for m in matches if m[0] is not None]
refSchema = matches[0][0].schema if matches else None
if self.cterm is not None and len(matches) != 0:
# Add a "flux" field to the input schema of the first element
# of the match and populate it with a colorterm correct flux.
mapper = afwTable.SchemaMapper(refSchema)
for key, field in refSchema:
mapper.addMapping(key)
fluxKey = mapper.editOutputSchema().addField("flux", type=float, doc="Reference flux")
fluxErrKey = mapper.editOutputSchema().addField("fluxErr", type=float,
doc="Reference flux uncertainty")
table = afwTable.SimpleTable.make(mapper.getOutputSchema())
table.preallocate(len(matches))
for match in matches:
newMatch = table.makeRecord()
newMatch.assign(match[0], mapper)
match[0] = newMatch
# extract the matched refCat as a Catalog for the colorterm code
refCat = afwTable.SimpleCatalog(matches[0].first.schema)
refCat.reserve(len(matches))
for x in matches:
record = refCat.addNew()
record.assign(x.first)
refMag, refMagErr = self.cterm.getCorrectedMagnitudes(refCat,
afwImage.Filter(calexp_md).getName())
# NOTE: mosaic assumes fluxes are in Jy
refFlux = (refMag*astropy.units.ABmag).to_value(astropy.units.Jy)
refFluxErr = afwImage.fluxErrFromABMagErr(refMagErr, refMag)
matches = [self.setCatFlux(m, flux, fluxKey, fluxErr, fluxErrKey) for
m, flux, fluxErr in zip(matches, refFlux, refFluxErr) if flux == flux]
else:
filterName = afwImage.Filter(calexp_md).getName()
refFluxField = measAlg.getRefFluxField(refSchema, filterName)
refSchema.getAliasMap().set("flux", refFluxField)
# LSST reads in a_net catalogs with flux in "janskys", so must convert back to DN.
matches = mosaicUtils.matchJanskyToDn(matches)
selSources = self.selectStars(sources, self.config.includeSaturated)
selMatches = self.selectStars(matches, self.config.includeSaturated)
retSrc = list()
retMatch = list()
if len(selMatches) > self.config.minNumMatch:
naxis1, naxis2 = afwImage.bboxFromMetadata(calexp_md).getDimensions()
if hscRun is None:
if nQuarter%2 != 0:
naxis1, naxis2 = naxis2, naxis1
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(naxis1, naxis2))
cellSet = afwMath.SpatialCellSet(bbox, self.config.cellSize, self.config.cellSize)
for s in selSources:
if numpy.isfinite(s.getRa().asDegrees()): # get rid of NaN
src = measMosaic.Source(s)
src.setExp(dataId["visit"])
src.setChip(dataId["ccd"])
try:
tmp = measMosaic.SpatialCellSource(src)
cellSet.insertCandidate(tmp)
except:
self.log.info("FAILED TO INSERT CANDIDATE: visit=%d ccd=%d x=%f y=%f" %
(dataRef.dataId["visit"], dataRef.dataId["ccd"],
src.getX(), src.getY()) + " bbox=" + str(bbox))
for cell in cellSet.getCellList():
cell.sortCandidates()
for i, cand in enumerate(cell):
src = cand.getSource()
retSrc.append(src)
if i == self.config.nStarPerCell - 1:
break
for m in selMatches:
if m[0] is not None and m[1] is not None:
match = (measMosaic.Source(m[0], wcs), measMosaic.Source(m[1]))
match[1].setExp(dataId["visit"])
match[1].setChip(dataId["ccd"])
retMatch.append(match)
else:
self.log.info("%8d %3d : %d/%d matches Suspicious to wrong match. Ignore this CCD" %
(dataRef.dataId["visit"], dataRef.dataId["ccd"], len(selMatches), len(matches)))
except Exception as e:
self.log.warn("Failed to read %s: %s" % (dataId, e))
return dataId, [None, None, None]
return dataId, [retSrc, retMatch, wcs]
def _computeReferenceOffsets(self, butler):
"""
Compute offsets relative to a reference catalog.
Parameters
----------
butler: lsst.daf.persistence.Butler
Returns
-------
offsets: np.array of floats
Per band zeropoint offsets
"""
# Load the stars
stars = butler.get('fgcmStandardStars', fgcmcycle=self.useCycle)
# Only use stars that are observed in all the bands
minObs = stars['ngood'].min(axis=1)
# Depending on how things work, this might need to be configurable
# However, I think that best results will occur when we use the same
# pixels to do the calibration for all the bands, so I think this
# is the right choice.
goodStars = (minObs >= 1)
stars = stars[goodStars]
self.log.info(
"Found %d stars with at least 1 good observation in each band" %
(len(stars)))
# We have to make a table for each pixel with flux/fluxErr
sourceMapper = afwTable.SchemaMapper(stars.schema)
sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
sourceMapper.editOutputSchema().addField('flux',
type=np.float64,
doc="flux")
sourceMapper.editOutputSchema().addField('fluxErr',
type=np.float64,
doc="flux error")
badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
type='Flag',
doc="bad flag")
# The exposure is used to record the filter name
exposure = afwImage.ExposureF()
# Split up the stars (units are radians)
theta = np.pi / 2. - stars['coord_dec']
phi = stars['coord_ra']
ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
h, rev = esutil.stat.histogram(ipring, rev=True)
gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
(gdpix.size, self.config.referencePixelizationNside,
self.config.referencePixelizationMinStars))
if gdpix.size < self.config.referencePixelizationNPixels:
self.log.warn(
"Found fewer good pixels (%d) than preferred in configuration (%d)"
% (gdpix.size, self.config.referencePixelizationNPixels))
else:
# Sample out the pixels we want to use
gdpix = np.random.choice(
gdpix,
size=self.config.referencePixelizationNPixels,
replace=False)
results = np.zeros(gdpix.size,
dtype=[('hpix', 'i4'),
('nstar', 'i4', len(self.bands)),
('nmatch', 'i4', len(self.bands)),
('zp', 'f4', len(self.bands)),
('zpErr', 'f4', len(self.bands))])
results['hpix'] = ipring[rev[rev[gdpix]]]
# We need a boolean index to deal with catalogs...
selected = np.zeros(len(stars), dtype=np.bool)
refFluxFields = [None] * len(self.bands)
for p, pix in enumerate(gdpix):
i1a = rev[rev[pix]:rev[pix + 1]]
# Need to convert to a boolean array
selected[:] = False
selected[i1a] = True
for b, band in enumerate(self.bands):
sourceCat = afwTable.SimpleCatalog(
sourceMapper.getOutputSchema())
sourceCat.reserve(len(i1a))
sourceCat.extend(stars[selected], mapper=sourceMapper)
sourceCat['flux'] = afwImage.fluxFromABMag(
stars['mag_std_noabs'][selected, b])
sourceCat['fluxErr'] = afwImage.fluxErrFromABMagErr(
stars['magerr_std'][selected, b],
stars['mag_std_noabs'][selected, b])
# Make sure we only use stars that have valid measurements
# (This is perhaps redundant with requirements above that the
# stars be observed in all bands, but it can't hurt)
badStar = (stars['mag_std_noabs'][selected, b] > 90.0)
for rec in sourceCat[badStar]:
rec.set(badStarKey, True)
exposure.setFilter(afwImage.Filter(band))
if refFluxFields[b] is None:
# Need to find the flux field in the reference catalog
# to work around limitations of DirectMatch in PhotoCal
ctr = stars[0].getCoord()
rad = 0.05 * lsst.geom.degrees
refDataTest = self.refObjLoader.loadSkyCircle(
ctr, rad, band)
refFluxFields[b] = refDataTest.fluxField
# Make a copy of the config so that we can modify it
calConfig = copy.copy(self.config.photoCal.value)
calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[
b]
calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[
b] + 'Err'
calTask = self.config.photoCal.target(
refObjLoader=self.refObjLoader,
config=calConfig,
schema=sourceCat.getSchema())
struct = calTask.run(exposure, sourceCat)
results['nstar'][p, b] = len(i1a)
results['nmatch'][p, b] = len(struct.arrays.refMag)
results['zp'][p, b] = struct.zp
results['zpErr'][p, b] = struct.sigma
# And compute the summary statistics
offsets = np.zeros(len(self.bands))
for b, band in enumerate(self.bands):
# make configurable
ok, = np.where(
results['nmatch'][:, b] >= self.config.referenceMinMatch)
offsets[b] = np.median(results['zp'][ok, b])
madSigma = 1.4826 * np.median(
np.abs(results['zp'][ok, b] - offsets[b]))
self.log.info(
"Reference catalog offset for %s band: %.6f +/- %.6f" %
(band, offsets[b], madSigma))
return offsets
def _runFgcmOutputProducts(self, visitDataRefName, ccdDataRefName,
filterMapping, zpOffsets, testVisit, testCcd,
testFilter, testBandIndex):
"""
"""
if self.logLevel is not None:
self.otherArgs.extend(['--loglevel', 'fgcmcal=%s' % self.logLevel])
args = [self.inputDir, '--output', self.testDir, '--doraise']
args.extend(self.otherArgs)
result = fgcmcal.FgcmOutputProductsTask.parseAndRun(
args=args, config=self.config, doReturnResults=True)
self._checkResult(result)
# Extract the offsets from the results
offsets = result.resultList[0].results.offsets
self.assertFloatsAlmostEqual(offsets[0], zpOffsets[0], rtol=1e-6)
self.assertFloatsAlmostEqual(offsets[1], zpOffsets[1], rtol=1e-6)
butler = dafPersistence.butler.Butler(self.testDir)
# Test the reference catalog stars
# Read in the raw stars...
rawStars = butler.get('fgcmStandardStars', fgcmcycle=0)
# Read in the new reference catalog...
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = 'fgcm_stars'
task = LoadIndexedReferenceObjectsTask(butler, config=config)
# Read in a giant radius to get them all
refStruct = task.loadSkyCircle(rawStars[0].getCoord(),
5.0 * lsst.geom.degrees,
filterName='r')
# Make sure all the stars are there
self.assertEqual(len(rawStars), len(refStruct.refCat))
# And make sure the numbers are consistent
test, = np.where(rawStars['id'][0] == refStruct.refCat['id'])
mag = rawStars['mag_std_noabs'][0, 0] + offsets[0]
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(rawStars['magerr_std'][0, 0],
mag)
self.assertFloatsAlmostEqual(flux,
refStruct.refCat['r_flux'][test[0]],
rtol=1e-6)
self.assertFloatsAlmostEqual(fluxErr,
refStruct.refCat['r_fluxErr'][test[0]],
rtol=1e-6)
# Test the joincal_photoCalib output
zptCat = butler.get('fgcmZeropoints', fgcmcycle=0)
selected = (zptCat['fgcmflag'] < 16)
# Read in all the calibrations, these should all be there
for rec in zptCat[selected]:
testCal = butler.get('jointcal_photoCalib',
dataId={
visitDataRefName: int(rec['visit']),
ccdDataRefName: int(rec['ccd']),
'filter':
filterMapping[rec['filtername']],
'tract': 0
})
# Our round-trip tests will be on this final one which is still loaded
testCal = butler.get('jointcal_photoCalib',
dataId={
visitDataRefName: int(testVisit),
ccdDataRefName: int(testCcd),
'filter': filterMapping[testFilter],
'tract': 0
})
src = butler.get('src',
dataId={
visitDataRefName: int(testVisit),
ccdDataRefName: int(testCcd)
})
# Only test sources with positive flux
gdSrc = (src['slot_CalibFlux_flux'] > 0.0)
# We need to apply the calibration offset to the fgcmzpt (which is internal
# and doesn't know about that yet)
testZpInd, = np.where((zptCat['visit'] == testVisit)
& (zptCat['ccd'] == testCcd))
fgcmZpt = zptCat['fgcmzpt'][testZpInd] + offsets[testBandIndex]
# This is the magnitude through the mean calibration
photoCalMeanCalMags = np.zeros(gdSrc.sum())
# This is the magnitude through the full focal-plane variable mags
photoCalMags = np.zeros_like(photoCalMeanCalMags)
# This is the magnitude with the FGCM (central-ccd) zeropoint
zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
for i, rec in enumerate(src[gdSrc]):
photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(
rec['slot_CalibFlux_flux'])
photoCalMags[i] = testCal.instFluxToMagnitude(
rec['slot_CalibFlux_flux'], rec.getCentroid())
zptMeanCalMags[i] = fgcmZpt - 2.5 * np.log10(
rec['slot_CalibFlux_flux'])
# These should be very close but some tiny differences because the fgcm value
# is defined at the center of the bbox, and the photoCal is the mean over the box
self.assertFloatsAlmostEqual(photoCalMeanCalMags,
zptMeanCalMags,
rtol=1e-6)
# These should be roughly equal, but not precisely because of the focal-plane
# variation. However, this is a useful sanity check for something going totally
# wrong.
self.assertFloatsAlmostEqual(photoCalMeanCalMags,
photoCalMags,
rtol=1e-2)
# Test the transmission output
visitCatalog = butler.get('fgcmVisitCatalog')
lutCat = butler.get('fgcmLookUpTable')
testTrans = butler.get(
'transmission_atmosphere_fgcm',
dataId={visitDataRefName: visitCatalog[0]['visit']})
testResp = testTrans.sampleAt(position=afwGeom.Point2D(0, 0),
wavelengths=lutCat[0]['atmlambda'])
# The fit to be roughly consistent with the standard, although the
# airmass is taken into account even with the "frozen" atmosphere.
# This is also a rough comparison, because the interpolation does
# not work well with such a coarse look-up table used for the test.
self.assertFloatsAlmostEqual(testResp,
lutCat[0]['atmstdtrans'],
atol=0.06)
# The second should be close to the first, but there is the airmass
# difference so they aren't identical
testTrans2 = butler.get(
'transmission_atmosphere_fgcm',
dataId={visitDataRefName: visitCatalog[1]['visit']})
testResp2 = testTrans2.sampleAt(position=afwGeom.Point2D(0, 0),
wavelengths=lutCat[0]['atmlambda'])
self.assertFloatsAlmostEqual(testResp, testResp2, atol=1e-4)
