def test_properMotion(self):
"""Test proper motion correction."""
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.filterMap = {'aprime': 'a'}
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
cat = loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a').refCat
# Zero epoch change --> no proper motion correction (except minor numerical effects)
cat_pm = loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a',
epoch=self.epoch).refCat
self.assertFloatsAlmostEqual(cat_pm['coord_ra'],
cat['coord_ra'],
rtol=1.0e-14)
self.assertFloatsAlmostEqual(cat_pm['coord_dec'],
cat['coord_dec'],
rtol=1.0e-14)
self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr'])
self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr'])
# One year difference
cat_pm = loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a',
epoch=self.epoch +
1.0 * astropy.units.yr).refCat
self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
for orig, ref in zip(cat, cat_pm):
self.assertAnglesAlmostEqual(orig.getCoord().separation(
ref.getCoord()),
self.properMotionAmt,
maxDiff=1.0e-6 * lsst.geom.arcseconds)
self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(
ref.getCoord()),
self.properMotionDir,
maxDiff=1.0e-4 * lsst.geom.arcseconds)
predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
def test_loadSkyCircle(self):
"""Test the loadSkyCircle routine."""
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs], self.handles)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
cat = loader.loadSkyCircle(
center,
30.0 * lsst.geom.degrees,
filterName='a',
).refCat
# Check that the max distance is less than the radius
dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']),
np.rad2deg(cat['coord_dec']))
self.assertLess(np.max(dist), 30.0)
# Check that all the objects from the two catalogs are here.
dist = sphdist(180.0, 0.0, self.skyCatalog['ra_icrs'],
self.skyCatalog['dec_icrs'])
inside, = (dist < 30.0).nonzero()
self.assertEqual(len(cat), len(inside))
self.assertTrue(cat.isContiguous())
self.assertEqual(len(np.unique(cat['id'])), len(cat))
# A default-loaded sky circle should not have centroids
self.assertNotIn('centroid_x', cat.schema)
self.assertNotIn('centroid_y', cat.schema)
self.assertNotIn('hasCentroid', cat.schema)
def test_requireProperMotion(self):
"""Tests of the requireProperMotion config field."""
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.requireProperMotion = True
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
# Test that we require an epoch set.
msg = 'requireProperMotion=True but epoch not provided to loader'
with self.assertRaisesRegex(RuntimeError, msg):
loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a')
def test_filterMap(self):
"""Test filterMap parameters."""
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.filterMap = {'aprime': 'a'}
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
result = loader.loadSkyCircle(
center,
30.0 * lsst.geom.degrees,
filterName='aprime',
)
self.assertEqual(result.fluxField, 'aprime_camFlux')
self.assertFloatsEqual(result.refCat['aprime_camFlux'],
result.refCat['a_flux'])
class FgcmOutputProductsTask(pipeBase.PipelineTask):
"""
Output products from FGCM global calibration.
"""
ConfigClass = FgcmOutputProductsConfig
_DefaultName = "fgcmOutputProducts"
def __init__(self, **kwargs):
super().__init__(**kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = {}
handleDict['camera'] = butlerQC.get(inputRefs.camera)
handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
handleDict['fgcmVisitCatalog'] = butlerQC.get(
inputRefs.fgcmVisitCatalog)
handleDict['fgcmStandardStars'] = butlerQC.get(
inputRefs.fgcmStandardStars)
if self.config.doZeropointOutput:
handleDict['fgcmZeropoints'] = butlerQC.get(
inputRefs.fgcmZeropoints)
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
if self.config.doAtmosphereOutput:
handleDict['fgcmAtmosphereParameters'] = butlerQC.get(
inputRefs.fgcmAtmosphereParameters)
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
if self.config.doReferenceCalibration:
refConfig = LoadReferenceObjectsConfig()
self.refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
log=self.log,
config=refConfig)
else:
self.refObjLoader = None
struct = self.run(handleDict, self.config.physicalFilterMap)
# Output the photoCalib exposure catalogs
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
# Output the atmospheres
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
if self.config.doReferenceCalibration:
# Turn offset into simple catalog for persistence if necessary
schema = afwTable.Schema()
schema.addField('offset',
type=np.float64,
doc="Post-process calibration offset (mag)")
offsetCat = afwTable.BaseCatalog(schema)
offsetCat.resize(len(struct.offsets))
offsetCat['offset'][:] = struct.offsets
butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
return
def run(self, handleDict, physicalFilterMap):
"""Run the output products task.
Parameters
----------
handleDict : `dict`
All handles are `lsst.daf.butler.DeferredDatasetHandle`
handle dictionary with keys:
``"camera"``
Camera object (`lsst.afw.cameraGeom.Camera`)
``"fgcmLookUpTable"``
handle for the FGCM look-up table.
``"fgcmVisitCatalog"``
handle for visit summary catalog.
``"fgcmStandardStars"``
handle for the output standard star catalog.
``"fgcmZeropoints"``
handle for the zeropoint data catalog.
``"fgcmAtmosphereParameters"``
handle for the atmosphere parameter catalog.
``"fgcmBuildStarsTableConfig"``
Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
physicalFilterMap : `dict`
Dictionary of mappings from physical filter to FGCM band.
Returns
-------
retStruct : `lsst.pipe.base.Struct`
Output structure with keys:
offsets : `np.ndarray`
Final reference offsets, per band.
atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
Generator that returns (visit, transmissionCurve) tuples.
photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
Generator that returns (visit, exposureCatalog) tuples.
"""
stdCat = handleDict['fgcmStandardStars'].get()
md = stdCat.getMetadata()
bands = md.getArray('BANDS')
if self.config.doReferenceCalibration:
lutCat = handleDict['fgcmLookUpTable'].get()
offsets = self._computeReferenceOffsets(stdCat, lutCat,
physicalFilterMap, bands)
else:
offsets = np.zeros(len(bands))
del stdCat
if self.config.doZeropointOutput:
zptCat = handleDict['fgcmZeropoints'].get()
visitCat = handleDict['fgcmVisitCatalog'].get()
pcgen = self._outputZeropoints(handleDict['camera'], zptCat,
visitCat, offsets, bands,
physicalFilterMap)
else:
pcgen = None
if self.config.doAtmosphereOutput:
atmCat = handleDict['fgcmAtmosphereParameters'].get()
atmgen = self._outputAtmospheres(handleDict, atmCat)
else:
atmgen = None
retStruct = pipeBase.Struct(offsets=offsets, atmospheres=atmgen)
retStruct.photoCalibCatalogs = pcgen
return retStruct
def generateTractOutputProducts(self, handleDict, tract, visitCat, zptCat,
atmCat, stdCat, fgcmBuildStarsConfig):
"""
Generate the output products for a given tract, as specified in the config.
This method is here to have an alternate entry-point for
FgcmCalibrateTract.
Parameters
----------
handleDict : `dict`
All handles are `lsst.daf.butler.DeferredDatasetHandle`
handle dictionary with keys:
``"camera"``
Camera object (`lsst.afw.cameraGeom.Camera`)
``"fgcmLookUpTable"``
handle for the FGCM look-up table.
tract : `int`
Tract number
visitCat : `lsst.afw.table.BaseCatalog`
FGCM visitCat from `FgcmBuildStarsTask`
zptCat : `lsst.afw.table.BaseCatalog`
FGCM zeropoint catalog from `FgcmFitCycleTask`
atmCat : `lsst.afw.table.BaseCatalog`
FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
stdCat : `lsst.afw.table.SimpleCatalog`
FGCM standard star catalog from `FgcmFitCycleTask`
fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
Configuration object from `FgcmBuildStarsTask`
Returns
-------
retStruct : `lsst.pipe.base.Struct`
Output structure with keys:
offsets : `np.ndarray`
Final reference offsets, per band.
atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
Generator that returns (visit, transmissionCurve) tuples.
photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
Generator that returns (visit, exposureCatalog) tuples.
"""
physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
md = stdCat.getMetadata()
bands = md.getArray('BANDS')
if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
"in fgcmBuildStarsTask.")
if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
self.log.warning(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set."
)
if self.config.doReferenceCalibration:
lutCat = handleDict['fgcmLookUpTable'].get()
offsets = self._computeReferenceOffsets(stdCat, lutCat, bands,
physicalFilterMap)
else:
offsets = np.zeros(len(bands))
if self.config.doZeropointOutput:
pcgen = self._outputZeropoints(handleDict['camera'], zptCat,
visitCat, offsets, bands,
physicalFilterMap)
else:
pcgen = None
if self.config.doAtmosphereOutput:
atmgen = self._outputAtmospheres(handleDict, atmCat)
else:
atmgen = None
retStruct = pipeBase.Struct(offsets=offsets, atmospheres=atmgen)
retStruct.photoCalibCatalogs = pcgen
return retStruct
def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap,
bands):
"""
Compute offsets relative to a reference catalog.
This method splits the star catalog into healpix pixels
and computes the calibration transfer for a sample of
these pixels to approximate the 'absolute' calibration
values (on for each band) to apply to transfer the
absolute scale.
Parameters
----------
stdCat : `lsst.afw.table.SimpleCatalog`
FGCM standard stars
lutCat : `lsst.afw.table.SimpleCatalog`
FGCM Look-up table
physicalFilterMap : `dict`
Dictionary of mappings from physical filter to FGCM band.
bands : `list` [`str`]
List of band names from FGCM output
Returns
-------
offsets : `numpy.array` of floats
Per band zeropoint offsets
"""
# Only use stars that are observed in all the bands that were actually used
# This will ensure that we use the same healpix pixels for the absolute
# calibration of each band.
minObs = stdCat['ngood'].min(axis=1)
goodStars = (minObs >= 1)
stdCat = stdCat[goodStars]
self.log.info(
"Found %d stars with at least 1 good observation in each band" %
(len(stdCat)))
# Associate each band with the appropriate physicalFilter and make
# filterLabels
filterLabels = []
lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
physicalFilterMapBands = list(physicalFilterMap.values())
physicalFilterMapFilters = list(physicalFilterMap.keys())
for band in bands:
# Find a physical filter associated from the band by doing
# a reverse lookup on the physicalFilterMap dict
physicalFilterMapIndex = physicalFilterMapBands.index(band)
physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
# Find the appropriate fgcm standard physicalFilter
lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
filterLabels.append(
afwImage.FilterLabel(band=band, physical=stdPhysicalFilter))
# We have to make a table for each pixel with flux/fluxErr
# This is a temporary table generated for input to the photoCal task.
# These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
# have had chromatic corrections applied to get to the standard system
# specified by the atmosphere/instrumental parameters), nor are they
# in Jansky (since they don't have a proper absolute calibration: the overall
# zeropoint is estimated from the telescope size, etc.)
sourceMapper = afwTable.SchemaMapper(stdCat.schema)
sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64, doc="instrumental flux (counts)")
sourceMapper.editOutputSchema().addField(
'instFluxErr',
type=np.float64,
doc="instrumental flux error (counts)")
badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
type='Flag',
doc="bad flag")
# Split up the stars
# Note that there is an assumption here that the ra/dec coords stored
# on-disk are in radians, and therefore that starObs['coord_ra'] /
# starObs['coord_dec'] return radians when used as an array of numpy float64s.
theta = np.pi / 2. - stdCat['coord_dec']
phi = stdCat['coord_ra']
ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
h, rev = esutil.stat.histogram(ipring, rev=True)
gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
(gdpix.size, self.config.referencePixelizationNside,
self.config.referencePixelizationMinStars))
if gdpix.size < self.config.referencePixelizationNPixels:
self.log.warning(
"Found fewer good pixels (%d) than preferred in configuration (%d)"
% (gdpix.size, self.config.referencePixelizationNPixels))
else:
# Sample out the pixels we want to use
gdpix = np.random.choice(
gdpix,
size=self.config.referencePixelizationNPixels,
replace=False)
results = np.zeros(gdpix.size,
dtype=[('hpix', 'i4'), ('nstar', 'i4', len(bands)),
('nmatch', 'i4', len(bands)),
('zp', 'f4', len(bands)),
('zpErr', 'f4', len(bands))])
results['hpix'] = ipring[rev[rev[gdpix]]]
# We need a boolean index to deal with catalogs...
selected = np.zeros(len(stdCat), dtype=bool)
refFluxFields = [None] * len(bands)
for p_index, pix in enumerate(gdpix):
i1a = rev[rev[pix]:rev[pix + 1]]
# the stdCat afwTable can only be indexed with boolean arrays,
# and not numpy index arrays (see DM-16497). This little trick
# converts the index array into a boolean array
selected[:] = False
selected[i1a] = True
for b_index, filterLabel in enumerate(filterLabels):
struct = self._computeOffsetOneBand(sourceMapper, badStarKey,
b_index, filterLabel,
stdCat, selected,
refFluxFields)
results['nstar'][p_index, b_index] = len(i1a)
results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
results['zp'][p_index, b_index] = struct.zp
results['zpErr'][p_index, b_index] = struct.sigma
# And compute the summary statistics
offsets = np.zeros(len(bands))
for b_index, band in enumerate(bands):
# make configurable
ok, = np.where(
results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
offsets[b_index] = np.median(results['zp'][ok, b_index])
# use median absolute deviation to estimate Normal sigma
# see https://en.wikipedia.org/wiki/Median_absolute_deviation
madSigma = 1.4826 * np.median(
np.abs(results['zp'][ok, b_index] - offsets[b_index]))
self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f", band,
offsets[b_index], madSigma)
return offsets
def _computeOffsetOneBand(self, sourceMapper, badStarKey, b_index,
filterLabel, stdCat, selected, refFluxFields):
"""
Compute the zeropoint offset between the fgcm stdCat and the reference
stars for one pixel in one band
Parameters
----------
sourceMapper : `lsst.afw.table.SchemaMapper`
Mapper to go from stdCat to calibratable catalog
badStarKey : `lsst.afw.table.Key`
Key for the field with bad stars
b_index : `int`
Index of the band in the star catalog
filterLabel : `lsst.afw.image.FilterLabel`
filterLabel with band and physical filter
stdCat : `lsst.afw.table.SimpleCatalog`
FGCM standard stars
selected : `numpy.array(dtype=bool)`
Boolean array of which stars are in the pixel
refFluxFields : `list`
List of names of flux fields for reference catalog
"""
sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
sourceCat.reserve(selected.sum())
sourceCat.extend(stdCat[selected], mapper=sourceMapper)
sourceCat['instFlux'] = 10.**(
stdCat['mag_std_noabs'][selected, b_index] / (-2.5))
sourceCat['instFluxErr'] = (np.log(10.) / 2.5) * (
stdCat['magErr_std'][selected, b_index] * sourceCat['instFlux'])
# Make sure we only use stars that have valid measurements
# (This is perhaps redundant with requirements above that the
# stars be observed in all bands, but it can't hurt)
badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
for rec in sourceCat[badStar]:
rec.set(badStarKey, True)
exposure = afwImage.ExposureF()
exposure.setFilterLabel(filterLabel)
if refFluxFields[b_index] is None:
# Need to find the flux field in the reference catalog
# to work around limitations of DirectMatch in PhotoCal
ctr = stdCat[0].getCoord()
rad = 0.05 * lsst.geom.degrees
refDataTest = self.refObjLoader.loadSkyCircle(
ctr, rad, filterLabel.bandLabel)
refFluxFields[b_index] = refDataTest.fluxField
# Make a copy of the config so that we can modify it
calConfig = copy.copy(self.config.photoCal.value)
calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[
b_index]
calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[
b_index] + 'Err'
calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
config=calConfig,
schema=sourceCat.getSchema())
struct = calTask.run(exposure, sourceCat)
return struct
def _formatCatalog(self, fgcmStarCat, offsets, bands):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets : `list` with len(self.bands) entries
Zeropoint offsets to apply
bands : `list` [`str`]
List of band names from FGCM output
Returns
-------
formattedCat: `lsst.afw.table.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nTotal' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' %
(band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(bands):
mag = fgcmStarCat['mag_std_noabs'][:, b].astype(
np.float64) + offsets[b]
# We want fluxes in nJy from calibrated AB magnitudes
# (after applying offset). Updated after RFC-549 and RFC-575.
flux = (mag * units.ABmag).to_value(units.nJy)
fluxErr = (np.log(10.) /
2.5) * flux * fgcmStarCat['magErr_std'][:, b].astype(
np.float64)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
formattedCat['%s_nPsfCandidate' %
(band)][:] = fgcmStarCat['npsfcand'][:, b]
addRefCatMetadata(formattedCat)
return formattedCat
def _outputZeropoints(self,
camera,
zptCat,
visitCat,
offsets,
bands,
physicalFilterMap,
tract=None):
"""Output the zeropoints in fgcm_photoCalib format.
Parameters
----------
camera : `lsst.afw.cameraGeom.Camera`
Camera from the butler.
zptCat : `lsst.afw.table.BaseCatalog`
FGCM zeropoint catalog from `FgcmFitCycleTask`.
visitCat : `lsst.afw.table.BaseCatalog`
FGCM visitCat from `FgcmBuildStarsTask`.
offsets : `numpy.array`
Float array of absolute calibration offsets, one for each filter.
bands : `list` [`str`]
List of band names from FGCM output.
physicalFilterMap : `dict`
Dictionary of mappings from physical filter to FGCM band.
tract: `int`, optional
Tract number to output. Default is None (global calibration)
Returns
-------
photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
Generator that returns (visit, exposureCatalog) tuples.
"""
# Select visit/ccds where we have a calibration
# This includes ccds where we were able to interpolate from neighboring
# ccds.
cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
& (zptCat['fgcmZptVar'] > 0.0)
& (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE))
# Log warnings for any visit which has no valid zeropoints
badVisits = np.unique(zptCat['visit'][~selected])
goodVisits = np.unique(zptCat['visit'][selected])
allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
for allBadVisit in allBadVisits:
self.log.warning(f'No suitable photoCalib for visit {allBadVisit}')
# Get a mapping from filtername to the offsets
offsetMapping = {}
for f in physicalFilterMap:
# Not every filter in the map will necesarily have a band.
if physicalFilterMap[f] in bands:
offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
# Get a mapping from "ccd" to the ccd index used for the scaling
ccdMapping = {}
for ccdIndex, detector in enumerate(camera):
ccdMapping[detector.getId()] = ccdIndex
# And a mapping to get the flat-field scaling values
scalingMapping = {}
for rec in visitCat:
scalingMapping[rec['visit']] = rec['scaling']
if self.config.doComposeWcsJacobian:
approxPixelAreaFields = computeApproxPixelAreaFields(camera)
# The zptCat is sorted by visit, which is useful
lastVisit = -1
zptVisitCatalog = None
metadata = dafBase.PropertyList()
metadata.add("COMMENT", "Catalog id is detector id, sorted.")
metadata.add("COMMENT", "Only detectors with data have entries.")
for rec in zptCat[selected]:
# Retrieve overall scaling
scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
# The postCalibrationOffset describe any zeropoint offsets
# to apply after the fgcm calibration. The first part comes
# from the reference catalog match (used in testing). The
# second part comes from the mean chromatic correction
# (if configured).
postCalibrationOffset = offsetMapping[rec['filtername']]
if self.config.doApplyMeanChromaticCorrection:
postCalibrationOffset += rec['fgcmDeltaChrom']
fgcmSuperStarField = self._getChebyshevBoundedField(
rec['fgcmfZptSstarCheb'], rec['fgcmfZptChebXyMax'])
# Convert from FGCM AB to nJy
fgcmZptField = self._getChebyshevBoundedField(
(rec['fgcmfZptCheb'] * units.AB).to_value(units.nJy),
rec['fgcmfZptChebXyMax'],
offset=postCalibrationOffset,
scaling=scaling)
if self.config.doComposeWcsJacobian:
fgcmField = afwMath.ProductBoundedField([
approxPixelAreaFields[rec['detector']], fgcmSuperStarField,
fgcmZptField
])
else:
# The photoCalib is just the product of the fgcmSuperStarField and the
# fgcmZptField
fgcmField = afwMath.ProductBoundedField(
[fgcmSuperStarField, fgcmZptField])
# The "mean" calibration will be set to the center of the ccd for reference
calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
calibErr = (np.log(10.0) / 2.5) * calibCenter * np.sqrt(
rec['fgcmZptVar'])
photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
calibrationErr=calibErr,
calibration=fgcmField,
isConstant=False)
# Return full per-visit exposure catalogs
if rec['visit'] != lastVisit:
# This is a new visit. If the last visit was not -1, yield
# the ExposureCatalog
if lastVisit > -1:
# ensure that the detectors are in sorted order, for fast lookups
zptVisitCatalog.sort()
yield (int(lastVisit), zptVisitCatalog)
else:
# We need to create a new schema
zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema(
)
zptExpCatSchema.addField('visit',
type='L',
doc='Visit number')
# And start a new one
zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
zptVisitCatalog.setMetadata(metadata)
lastVisit = int(rec['visit'])
catRecord = zptVisitCatalog.addNew()
catRecord['id'] = int(rec['detector'])
catRecord['visit'] = rec['visit']
catRecord.setPhotoCalib(photoCalib)
# Final output of last exposure catalog
# ensure that the detectors are in sorted order, for fast lookups
zptVisitCatalog.sort()
yield (int(lastVisit), zptVisitCatalog)
def _getChebyshevBoundedField(self,
coefficients,
xyMax,
offset=0.0,
scaling=1.0):
"""
Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
and scaling.
Parameters
----------
coefficients: `numpy.array`
Flattened array of chebyshev coefficients
xyMax: `list` of length 2
Maximum x and y of the chebyshev bounding box
offset: `float`, optional
Absolute calibration offset. Default is 0.0
scaling: `float`, optional
Flat scaling value from fgcmBuildStars. Default is 1.0
Returns
-------
boundedField: `lsst.afw.math.ChebyshevBoundedField`
"""
orderPlus1 = int(np.sqrt(coefficients.size))
pars = np.zeros((orderPlus1, orderPlus1))
bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
lsst.geom.Point2I(*xyMax))
pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
(10.**(offset / -2.5)) * scaling)
boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
return boundedField
def _outputAtmospheres(self, handleDict, atmCat):
"""
Output the atmospheres.
Parameters
----------
handleDict : `dict`
All data handles are `lsst.daf.butler.DeferredDatasetHandle`
The handleDict has the follownig keys:
``"fgcmLookUpTable"``
handle for the FGCM look-up table.
atmCat : `lsst.afw.table.BaseCatalog`
FGCM atmosphere parameter catalog from fgcmFitCycleTask.
Returns
-------
atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
Generator that returns (visit, transmissionCurve) tuples.
"""
# First, we need to grab the look-up table and key info
lutCat = handleDict['fgcmLookUpTable'].get()
atmosphereTableName = lutCat[0]['tablename']
elevation = lutCat[0]['elevation']
atmLambda = lutCat[0]['atmLambda']
lutCat = None
# Make the atmosphere table if possible
try:
atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(
atmosphereTableName)
atmTable.loadTable()
except IOError:
atmTable = None
if atmTable is None:
# Try to use MODTRAN instead
try:
modGen = fgcm.ModtranGenerator(elevation)
lambdaRange = np.array([atmLambda[0], atmLambda[-1]]) / 10.
lambdaStep = (atmLambda[1] - atmLambda[0]) / 10.
except (ValueError, IOError) as e:
raise RuntimeError(
"FGCM look-up-table generated with modtran, "
"but modtran not configured to run.") from e
zenith = np.degrees(np.arccos(1. / atmCat['secZenith']))
for i, visit in enumerate(atmCat['visit']):
if atmTable is not None:
# Interpolate the atmosphere table
atmVals = atmTable.interpolateAtmosphere(
pmb=atmCat[i]['pmb'],
pwv=atmCat[i]['pwv'],
o3=atmCat[i]['o3'],
tau=atmCat[i]['tau'],
alpha=atmCat[i]['alpha'],
zenith=zenith[i],
ctranslamstd=[atmCat[i]['cTrans'], atmCat[i]['lamStd']])
else:
# Run modtran
modAtm = modGen(
pmb=atmCat[i]['pmb'],
pwv=atmCat[i]['pwv'],
o3=atmCat[i]['o3'],
tau=atmCat[i]['tau'],
alpha=atmCat[i]['alpha'],
zenith=zenith[i],
lambdaRange=lambdaRange,
lambdaStep=lambdaStep,
ctranslamstd=[atmCat[i]['cTrans'], atmCat[i]['lamStd']])
atmVals = modAtm['COMBINED']
# Now need to create something to persist...
curve = TransmissionCurve.makeSpatiallyConstant(
throughput=atmVals,
wavelengths=atmLambda,
throughputAtMin=atmVals[0],
throughputAtMax=atmVals[-1])
yield (int(visit), curve)
class LoadReferenceCatalogTask(pipeBase.Task):
"""Load multi-band reference objects from a reference catalog.
Parameters
----------
dataIds : iterable of `lsst.daf.butler.dataId`, optional
An iterable object of dataIds which point to reference catalogs
in a Gen3 repository. Required for Gen3.
refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`, optional
An iterable object of dataset refs for reference catalogs in
a Gen3 repository. Required for Gen3.
butler : `lsst.daf.persistence.Butler`, optional
A Gen2 butler. Required for Gen2.
"""
ConfigClass = LoadReferenceCatalogConfig
_DefaultName = "loadReferenceCatalog"
def __init__(self, dataIds=None, refCats=None, butler=None, **kwargs):
if dataIds is not None and refCats is not None:
pipeBase.Task.__init__(self, **kwargs)
refConfig = self.config.refObjLoader
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
elif butler is not None:
raise RuntimeError("LoadReferenceCatalogTask does not support Gen2 Butlers.")
else:
raise RuntimeError("Must instantiate LoadReferenceCatalogTask with "
"dataIds and refCats (Gen3)")
if self.config.doReferenceSelection:
self.makeSubtask('referenceSelector')
self._fluxFilters = None
self._fluxFields = None
self._referenceFilter = None
def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None,
bboxToSpherePadding=None):
"""Get a multi-band reference catalog by specifying a bounding box and WCS.
The catalog will be in `numpy.ndarray`, with positions proper-motion
corrected to "epoch" (if specified, and if the reference catalog has
proper motions); sources cut on a reference selector (if
"config.doReferenceSelection = True"); and color-terms applied (if
"config.doApplyColorTerms = True").
The format of the reference catalog will be of the format:
dtype = [('ra', 'np.float64'),
('dec', 'np.float64'),
('refMag', 'np.float32', (len(filterList), )),
('refMagErr', 'np.float32', (len(filterList), ))]
Reference magnitudes (AB) and errors will be 99 for non-detections
in a given band.
Parameters
----------
bbox : `lsst.geom.Box2I`
Box which bounds a region in pixel space.
wcs : `lsst.afw.geom.SkyWcs`
Wcs object defining the pixel to sky (and reverse) transform for
the supplied bbox.
filterList : `List` [ `str` ]
List of camera physicalFilter names to retrieve magnitudes.
epoch : `astropy.time.Time`, optional
Epoch to which to correct proper motion and parallax
(if available), or `None` to not apply such corrections.
bboxToSpherePadding : `int`, optional
Padding to account for translating a set of corners into a
spherical (convex) boundary that is certain to encompass the
entire area covered by the bbox.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
# Check if we have previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
center = wcs.pixelToSky(bbox.getCenter())
self._determineFluxFields(center, filterList)
skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter,
epoch=epoch,
bboxToSpherePadding=bboxToSpherePadding)
if not skyBox.refCat.isContiguous():
refCat = skyBox.refCat.copy(deep=True)
else:
refCat = skyBox.refCat
return self._formatCatalog(refCat, filterList)
def getSkyCircleCatalog(self, center, radius, filterList, epoch=None,
catalogFormat='numpy'):
"""Get a multi-band reference catalog by specifying a center and radius.
The catalog will be in `numpy.ndarray`, with positions proper-motion
corrected to "epoch" (if specified, and if the reference catalog has
proper motions); sources cut on a reference selector (if
"config.doReferenceSelection = True"); and color-terms applied (if
"config.doApplyColorTerms = True").
The format of the reference catalog will be of the format:
dtype = [('ra', 'np.float64'),
('dec', 'np.float64'),
('refMag', 'np.float32', (len(filterList), )),
('refMagErr', 'np.float32', (len(filterList), ))]
Reference magnitudes (AB) and errors will be 99 for non-detections
in a given band.
Parameters
----------
center : `lsst.geom.SpherePoint`
Point defining the center of the circular region.
radius : `lsst.geom.Angle`
Defines the angular radius of the circular region.
filterList : `List` [ `str` ]
List of camera physicalFilter names to retrieve magnitudes.
epoch : `astropy.time.Time`, optional
Epoch to which to correct proper motion and parallax
(if available), or `None` to not apply such corrections.
Parameters
----------
refCat : `numpy.ndarray`
Reference catalog.
"""
# Check if we have previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
self._determineFluxFields(center, filterList)
skyCircle = self.refObjLoader.loadSkyCircle(center, radius,
self._referenceFilter,
epoch=epoch)
if not skyCircle.refCat.isContiguous():
refCat = skyCircle.refCat.copy(deep=True)
else:
refCat = skyCircle.refCat
return self._formatCatalog(refCat, filterList)
def _formatCatalog(self, refCat, filterList):
"""Format a reference afw table into the final format.
This method applies reference selections and color terms as specified
by the config.
Parameters
----------
refCat : `lsst.afw.table.SourceCatalog`
Reference catalog in afw format.
filterList : `list` [`str`]
List of camera physicalFilter names to apply color terms.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
if self.config.doReferenceSelection:
goodSources = self.referenceSelector.selectSources(refCat)
selected = goodSources.selected
else:
selected = np.ones(len(refCat), dtype=bool)
npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'),
('dec', 'f8'),
('refMag', 'f4', (len(filterList), )),
('refMagErr', 'f4', (len(filterList), ))])
if npRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources.
return npRefCat
# Natively "coord_ra" and "coord_dec" are stored in radians.
# Doing this as an array rather than by row with the coord access is
# approximately 600x faster.
npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected])
npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected])
# Default (unset) values are 99.0
npRefCat['refMag'][:, :] = 99.0
npRefCat['refMagErr'][:, :] = 99.0
if self.config.doApplyColorTerms:
if isinstance(self.refObjLoader, ReferenceObjectLoader):
# Gen3
refCatName = self.refObjLoader.config.value.ref_dataset_name
else:
# Gen2
refCatName = self.refObjLoader.ref_dataset_name
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
if fluxField is None:
# There is no matching reference band.
# This will leave the column filled with 99s
continue
self.log.debug("Applying color terms for filterName='%s'", filterName)
colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True)
refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
# nan_to_num replaces nans with zeros, and this ensures
# that we select magnitudes that both filter out nans and are
# not very large (corresponding to very small fluxes), as "99"
# is a commen sentinel for illegal magnitudes.
good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected]) > 0.0))
npRefCat['refMag'][good, i] = refMag[selected][good]
npRefCat['refMagErr'][good, i] = refMagErr[selected][good]
else:
# No color terms to apply
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
# nan_to_num replaces nans with zeros, and this ensures that
# we select fluxes that both filter out nans and are positive.
good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0)
& (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0))
refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag)
refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good],
refCat[fluxField][selected][good])
npRefCat['refMag'][good, i] = refMag
npRefCat['refMagErr'][good, i] = refMagErr
return npRefCat
def _determineFluxFields(self, center, filterList):
"""Determine the flux field names for a reference catalog.
This method sets self._fluxFields, self._referenceFilter.
Parameters
----------
center : `lsst.geom.SpherePoint`
The center around which to load test sources.
filterList : `list` [`str`]
List of camera physicalFilter names.
"""
# Search for a good filter to use to load the reference catalog
# via the refObjLoader task which requires a valid filterName
foundReferenceFilter = False
# Store the original config
_config = self.refObjLoader.config
configTemp = LoadReferenceObjectsConfig()
configIntersection = {k: getattr(self.refObjLoader.config, k)
for k, v in self.refObjLoader.config.toDict().items()
if (k in configTemp.keys() and k != "connections")}
# We must turn off the proper motion checking to find the refFilter.
configIntersection['requireProperMotion'] = False
configTemp.update(**configIntersection)
self.refObjLoader.config = configTemp
for filterName in filterList:
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is None:
continue
try:
results = self.refObjLoader.loadSkyCircle(center,
0.05*lsst.geom.degrees,
refFilterName)
foundReferenceFilter = True
self._referenceFilter = refFilterName
break
except RuntimeError as err:
# This just means that the filterName wasn't listed
# in the reference catalog. This is okay.
if 'not find flux' in err.args[0]:
# The filterName wasn't listed in the reference catalog.
# This is not a fatal failure (yet)
pass
else:
raise err
self.refObjLoader.config = _config
if not foundReferenceFilter:
raise RuntimeError("Could not find any valid flux field(s) %s" %
(", ".join(filterList)))
# Record self._fluxFilters for checks on subsequent calls
self._fluxFilters = filterList
# Retrieve all the fluxField names
self._fluxFields = []
for filterName in filterList:
fluxField = None
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is not None:
try:
fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName)
except RuntimeError:
# This flux field isn't available. Set to None
fluxField = None
if fluxField is None:
self.log.warning('No reference flux field for camera filter %s', filterName)
self._fluxFields.append(fluxField)