def getCorrectedMagnitudes(self, refCat, filterName):
"""Return the colorterm corrected magnitudes for a given filter.
Parameters
----------
refCat : `lsst.afw.table.SimpleCatalog`
The reference catalog to apply color corrections to.
filterName : `str`
The camera filter to correct the reference catalog into.
Returns
-------
RefMag : `np.ndarray`
The corrected AB magnitudes.
RefMagErr : `np.ndarray`
The corrected AB magnitude errors.
Raises
------
KeyError
Raised if the reference catalog does not have a flux uncertainty
for that filter.
Notes
-----
WARNING: I do not know that we can trust the propagation of magnitude
errors returned by this method. They need more thorough tests.
"""
def getFluxes(fluxField):
"""Get the flux and fluxErr of this field from refCat."""
fluxKey = refCat.schema.find(fluxField).key
refFlux = refCat[fluxKey]
try:
fluxErrKey = refCat.schema.find(fluxField + "Err").key
refFluxErr = refCat[fluxErrKey]
except KeyError as e:
raise KeyError(
"Reference catalog does not have flux uncertainties for %s"
% fluxField) from e
return refFlux, refFluxErr
primaryFlux, primaryErr = getFluxes(self.primary + "_flux")
secondaryFlux, secondaryErr = getFluxes(self.secondary + "_flux")
primaryMag = u.Quantity(primaryFlux, u.nJy).to_value(u.ABmag)
secondaryMag = u.Quantity(secondaryFlux, u.nJy).to_value(u.ABmag)
refMag = self.transformMags(primaryMag, secondaryMag)
refFluxErrArr = self.propagateFluxErrors(primaryErr, secondaryErr)
# HACK convert to Jy until we have a replacement for this (DM-16903)
refMagErr = abMagErrFromFluxErr(refFluxErrArr * 1e-9,
primaryFlux * 1e-9)
return refMag, refMagErr
def testBasics(self):
for flux in (1, 210, 3210, 43210, 543210):
abMag = afwImage.abMagFromFlux(flux)
self.assertAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertAlmostEqual(flux, fluxRoundTrip)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertAlmostEqual(fluxErr, fluxErrRoundTrip)
def testVector(self):
flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0])
flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0>
abMag = afwImage.abMagFromFlux(flux)
self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertFloatsAlmostEqual(abMagErr, refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertFloatsAlmostEqual(fluxErr, fluxErrRoundTrip, rtol=1.0e-15)
def testVector(self):
flux = np.array([1.0, 210.0, 3210.0, 43210.0, 543210.0])
flux.flags.writeable = False # Put the 'const' into ndarray::Array<double const, 1, 0>
abMag = afwImage.abMagFromFlux(flux)
self.assertFloatsAlmostEqual(abMag, refABMagFromFlux(flux))
fluxRoundTrip = afwImage.fluxFromABMag(abMag)
self.assertFloatsAlmostEqual(flux, fluxRoundTrip, rtol=1.0e-15)
for fluxErrFrac in (0.001, 0.01, 0.1):
fluxErr = flux * fluxErrFrac
abMagErr = afwImage.abMagErrFromFluxErr(fluxErr, flux)
self.assertFloatsAlmostEqual(abMagErr,
refABMagErrFromFluxErr(fluxErr, flux))
fluxErrRoundTrip = afwImage.fluxErrFromABMagErr(abMagErr, abMag)
self.assertFloatsAlmostEqual(fluxErr,
fluxErrRoundTrip,
rtol=1.0e-15)
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
\param[in] matches Reference/source matches, a \link lsst::afw::table::ReferenceMatchVector\endlink
\param[in] filterName Name of filter being calibrated
\param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
\return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
\note These magnitude arrays are the \em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
srcFluxErrArr = np.array([m.second.get(sourceKeys.fluxErr) for m in matches])
if not np.all(np.isfinite(srcFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
srcFluxErrArr = np.sqrt(srcFluxArr)
# convert source flux from DN to an estimate of Jy
JanskysPerABFlux = 3631.0
srcFluxArr = srcFluxArr * JanskysPerABFlux
srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
applyCTReason += " and photoRefCat %s None" % ("is not" if photoCatSpecified else "is")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s" %
(filterName, self.config.photoCatName, applyCTReason))
ct = self.config.colorterms.getColorterm(
filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
else:
self.log.info("Not applying color terms because %s" % (applyCTReason,))
ct = None
if ct: # we have a color term to worry about
fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (ct.primary, ct.secondary)]
missingFluxFieldList = []
for fluxField in fluxFieldList:
try:
refSchema.find(fluxField).key
except KeyError:
missingFluxFieldList.append(fluxField)
if missingFluxFieldList:
self.log.warn("Source catalog does not have fluxes for %s; ignoring color terms" %
" ".join(missingFluxFieldList))
ct = None
if not ct:
fluxFieldList = [getRefFluxField(refSchema, filterName)]
refFluxArrList = [] # list of ref arrays, one per flux field
refFluxErrArrList = [] # list of ref flux arrays, one per flux field
for fluxField in fluxFieldList:
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Sigma").key
refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK
self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux)."
% fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refFluxArrList.append(refFluxArr)
refFluxErrArrList.append(refFluxErrArr)
if ct: # we have a color term to worry about
refMagArr1 = np.array([abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
refMagArr2 = np.array([abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
refMagArr = ct.transformMags(refMagArr1, refMagArr2)
refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0], refFluxErrArrList[1])
else:
refMagArr = np.array([abMagFromFlux(rf) for rf in refFluxArrList[0]])
srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
magErrArr = np.array([abMagErrFromFluxErr(fe, sf) for fe, sf in izip(srcFluxErrArr, srcFluxArr)])
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = np.array([abMagErrFromFluxErr(sfe, sf) for sfe, sf in izip(srcFluxErrArr, srcFluxArr)])
refMagErrArr = np.array([abMagErrFromFluxErr(rfe, rf) for rfe, rf in izip(refFluxErrArr, refFluxArr)])
return pipeBase.Struct(
srcMag = srcMagArr,
refMag = refMagArr,
magErr = magErrArr,
srcMagErr = srcMagErrArr,
refMagErr = refMagErrArr,
refFluxFieldList = fluxFieldList,
)
def getFgcmReferenceStarsSkyCircle(self, ra, dec, radius, filterList):
"""
Get a reference catalog that overlaps a circular sky region, using
multiple filters. In addition, apply colorterms if available.
Return format is a numpy recarray for use with fgcm.
Parameters
----------
ra: `float`
ICRS right ascension, degrees.
dec: `float`
ICRS declination, degrees.
radius: `float`
Radius to search, degrees.
filterList: `list`
list of `str` of camera filter names.
Returns
-------
fgcmRefCat: `np.recarray`
Numpy recarray with the following fields:
ra: `np.float64`
Right ascension, degrees
dec: `np.float64`
Declination, degrees
refMag: (`np.float32`, len(filterList))
Reference magnitude for filterList bands
Will be 99 for non-detections.
refMagErr: (`np.float32`, len(filterList))
Reference magnitude error for filterList bands
Will be 99 for non-detections.
"""
center = lsst.geom.SpherePoint(ra * lsst.geom.degrees,
dec * lsst.geom.degrees)
# Check if we haev previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
self._determine_flux_fields(center, filterList)
skyCircle = self.refObjLoader.loadSkyCircle(center,
radius * lsst.geom.degrees,
self._referenceFilter)
if not skyCircle.refCat.isContiguous():
refCat = skyCircle.refCat.copy(deep=True)
else:
refCat = skyCircle.refCat
# Select on raw (uncorrected) catalog, where the errors should make more sense
goodSources = self.referenceSelector.selectSources(refCat)
selected = goodSources.selected
fgcmRefCat = np.zeros(np.sum(selected),
dtype=[('ra', 'f8'), ('dec', 'f8'),
('refMag', 'f4', len(filterList)),
('refMagErr', 'f4', len(filterList))])
if fgcmRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources
return fgcmRefCat
# The ra/dec native Angle format is radians
# We determine the conversion from the native units (typically
# radians) to degrees for the first observation. This allows us
# to treate ra/dec as numpy arrays rather than Angles, which would
# be approximately 600x slower.
conv = refCat[0]['coord_ra'].asDegrees() / float(refCat[0]['coord_ra'])
fgcmRefCat['ra'] = refCat['coord_ra'][selected] * conv
fgcmRefCat['dec'] = refCat['coord_dec'][selected] * conv
# Default (unset) values are 99.0
fgcmRefCat['refMag'][:, :] = 99.0
fgcmRefCat['refMagErr'][:, :] = 99.0
if self.config.applyColorTerms:
try:
refCatName = self.refObjLoader.ref_dataset_name
except AttributeError:
# NOTE: we need this try:except: block in place until we've
# completely removed a.net support
raise RuntimeError(
"Cannot perform colorterm corrections with a.net refcats.")
for i, (filterName, fluxField) in enumerate(
zip(self._fluxFilters, self._fluxFields)):
if fluxField is None:
continue
self.log.debug("Applying color terms for filtername=%r" %
(filterName))
colorterm = self.config.colorterms.getColorterm(
filterName=filterName,
photoCatName=refCatName,
doRaise=True)
refMag, refMagErr = colorterm.getCorrectedMagnitudes(
refCat, filterName)
# nan_to_num replaces nans with zeros, and this ensures that we select
# magnitudes that both filter out nans and are not very large (corresponding
# to very small fluxes), as "99" is a common sentinel for illegal magnitudes.
good, = np.where((np.nan_to_num(refMag[selected]) < 90.0)
& (np.nan_to_num(refMagErr[selected]) < 90.0)
& (np.nan_to_num(refMagErr[selected]) > 0.0))
fgcmRefCat['refMag'][good, i] = refMag[selected][good]
fgcmRefCat['refMagErr'][good, i] = refMagErr[selected][good]
else:
# No colorterms
# TODO: need to use Jy here until RFC-549 is completed and refcats return nanojansky
for i, (filterName, fluxField) in enumerate(
zip(self._fluxFilters, self._fluxFields)):
# nan_to_num replaces nans with zeros, and this ensures that we select
# fluxes that both filter out nans and are positive.
good, = np.where(
(np.nan_to_num(refCat[fluxField][selected]) > 0.0) &
(np.nan_to_num(refCat[fluxField + 'Err'][selected]) > 0.0))
refMag = (refCat[fluxField][selected][good] *
units.Jy).to_value(units.ABmag)
refMagErr = abMagErrFromFluxErr(
refCat[fluxField + 'Err'][selected][good],
refCat[fluxField][selected][good])
fgcmRefCat['refMag'][good, i] = refMag
fgcmRefCat['refMagErr'][good, i] = refMagErr
return fgcmRefCat
def extractMagArrays(self, matches, filterLabel, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterLabel Label of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcInstFluxArr = np.array(
[m.second.get(sourceKeys.instFlux) for m in matches])
srcInstFluxErrArr = np.array(
[m.second.get(sourceKeys.instFluxErr) for m in matches])
if not np.all(np.isfinite(srcInstFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux)."
)
srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
# convert source instFlux from DN to an estimate of nJy
referenceFlux = (0 * u.ABmag).to_value(u.nJy)
srcInstFluxArr = srcInstFluxArr * referenceFlux
srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (
self.config.applyColorTerms, )
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail
else "is not")
applyCTReason += " and photoRefCat %s provided" % (
"is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info(
"Applying color terms for filter=%r, config.photoCatName=%s because %s",
filterLabel.physicalLabel, self.config.photoCatName,
applyCTReason)
colorterm = self.config.colorterms.getColorterm(
filterLabel.physicalLabel,
self.config.photoCatName,
doRaise=True)
refCat = afwTable.SimpleCatalog(matches[0].first.schema)
# extract the matched refCat as a Catalog for the colorterm code
refCat.reserve(len(matches))
for x in matches:
record = refCat.addNew()
record.assign(x.first)
refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat)
fluxFieldList = [
getRefFluxField(refSchema, filt)
for filt in (colorterm.primary, colorterm.secondary)
]
else:
# no colorterms to apply
self.log.info("Not applying color terms because %s", applyCTReason)
colorterm = None
fluxFieldList = [getRefFluxField(refSchema, filterLabel.bandLabel)]
fluxField = getRefFluxField(refSchema, filterLabel.bandLabel)
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Err").key
refFluxErrArr = np.array(
[m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK DM-2308
self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
# HACK convert to Jy until we have a replacement for this (DM-16903)
refMagErrArr = abMagErrFromFluxErr(refFluxErrArr * 1e-9,
refFluxArr * 1e-9)
# compute the source catalog magnitudes and errors
srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
# HACK convert to Jy until we have a replacement for this (DM-16903)
magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr * 1e-9,
srcInstFluxArr * 1e-9)
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr * 1e-9,
srcInstFluxArr * 1e-9)
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
hdu[1].data['i'],
hdu[1].data['z'],
hdu[1].data['y'],
hdu[1].data['g_err'],
hdu[1].data['r_err'],
hdu[1].data['i_err'],
hdu[1].data['z_err'],
hdu[1].data['y_err']))
areNans = np.logical_or.reduce(areNans)
hdu[1].data = hdu[1].data[~areNans]
#Limit to brighter than 19th mag in g-band
bright = hdu[1].data['g'] > 9.12e-5
hdu[1].data = hdu[1].data[bright]
hdu[1].data['g_err'] = abMagErrFromFluxErr(hdu[1].data['g_err'].astype(np.float),
hdu[1].data['g'].astype(np.float))
hdu[1].data['r_err'] = abMagErrFromFluxErr(hdu[1].data['r_err'].astype(np.float),
hdu[1].data['r'].astype(np.float))
hdu[1].data['i_err'] = abMagErrFromFluxErr(hdu[1].data['i_err'].astype(np.float),
hdu[1].data['i'].astype(np.float))
hdu[1].data['z_err'] = abMagErrFromFluxErr(hdu[1].data['z_err'].astype(np.float),
hdu[1].data['z'].astype(np.float))
hdu[1].data['y_err'] = abMagErrFromFluxErr(hdu[1].data['y_err'].astype(np.float),
hdu[1].data['y'].astype(np.float))
hdu[1].data['g'] = abMagFromFlux(hdu[1].data['g'].astype(np.float))
hdu[1].data['r'] = abMagFromFlux(hdu[1].data['r'].astype(np.float))
hdu[1].data['i'] = abMagFromFlux(hdu[1].data['i'].astype(np.float))
hdu[1].data['z'] = abMagFromFlux(hdu[1].data['z'].astype(np.float))
hdu[1].data['y'] = abMagFromFlux(hdu[1].data['y'].astype(np.float))
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterName Name of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcFluxArr = np.array([m.second.get(sourceKeys.flux) for m in matches])
srcFluxErrArr = np.array(
[m.second.get(sourceKeys.fluxErr) for m in matches])
if not np.all(np.isfinite(srcFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn(
"Source catalog does not have flux uncertainties; using sqrt(flux)."
)
srcFluxErrArr = np.sqrt(srcFluxArr)
# convert source flux from DN to an estimate of Jy
JanskysPerABFlux = 3631.0
srcFluxArr = srcFluxArr * JanskysPerABFlux
srcFluxErrArr = srcFluxErrArr * JanskysPerABFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (
self.config.applyColorTerms, )
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail
else "is not")
applyCTReason += " and photoRefCat %s provided" % (
"is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info(
"Applying color terms for filterName=%r, config.photoCatName=%s because %s",
filterName, self.config.photoCatName, applyCTReason)
ct = self.config.colorterms.getColorterm(
filterName=filterName,
photoCatName=self.config.photoCatName,
doRaise=True)
else:
self.log.info("Not applying color terms because %s", applyCTReason)
ct = None
if ct: # we have a color term to worry about
fluxFieldList = [
getRefFluxField(refSchema, filt)
for filt in (ct.primary, ct.secondary)
]
missingFluxFieldList = []
for fluxField in fluxFieldList:
try:
refSchema.find(fluxField).key
except KeyError:
missingFluxFieldList.append(fluxField)
if missingFluxFieldList:
self.log.warn(
"Source catalog does not have fluxes for %s; ignoring color terms",
" ".join(missingFluxFieldList))
ct = None
if not ct:
fluxFieldList = [getRefFluxField(refSchema, filterName)]
refFluxArrList = [] # list of ref arrays, one per flux field
refFluxErrArrList = [] # list of ref flux arrays, one per flux field
for fluxField in fluxFieldList:
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Sigma").key
refFluxErrArr = np.array(
[m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK
self.log.warn(
"Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refFluxArrList.append(refFluxArr)
refFluxErrArrList.append(refFluxErrArr)
if ct: # we have a color term to worry about
refMagArr1 = np.array(
[abMagFromFlux(rf1) for rf1 in refFluxArrList[0]]) # primary
refMagArr2 = np.array(
[abMagFromFlux(rf2) for rf2 in refFluxArrList[1]]) # secondary
refMagArr = ct.transformMags(refMagArr1, refMagArr2)
refFluxErrArr = ct.propagateFluxErrors(refFluxErrArrList[0],
refFluxErrArrList[1])
else:
refMagArr = np.array(
[abMagFromFlux(rf) for rf in refFluxArrList[0]])
srcMagArr = np.array([abMagFromFlux(sf) for sf in srcFluxArr])
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
magErrArr = np.array([
abMagErrFromFluxErr(fe, sf)
for fe, sf in zip(srcFluxErrArr, srcFluxArr)
])
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = np.array([
abMagErrFromFluxErr(sfe, sf)
for sfe, sf in zip(srcFluxErrArr, srcFluxArr)
])
refMagErrArr = np.array([
abMagErrFromFluxErr(rfe, rf)
for rfe, rf in zip(refFluxErrArr, refFluxArr)
])
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
def extractMagArrays(self, matches, filterName, sourceKeys):
"""!Extract magnitude and magnitude error arrays from the given matches.
@param[in] matches Reference/source matches, a @link lsst::afw::table::ReferenceMatchVector@endlink
@param[in] filterName Name of filter being calibrated
@param[in] sourceKeys Struct of source catalog keys, as returned by getSourceKeys()
@return Struct containing srcMag, refMag, srcMagErr, refMagErr, and magErr numpy arrays
where magErr is an error in the magnitude; the error in srcMag - refMag
If nonzero, config.magErrFloor will be added to magErr *only* (not srcMagErr or refMagErr), as
magErr is what is later used to determine the zero point.
Struct also contains refFluxFieldList: a list of field names of the reference catalog used for fluxes
(1 or 2 strings)
@note These magnitude arrays are the @em inputs to the photometric calibration, some may have been
discarded by clipping while estimating the calibration (https://jira.lsstcorp.org/browse/DM-813)
"""
srcInstFluxArr = np.array([m.second.get(sourceKeys.instFlux) for m in matches])
srcInstFluxErrArr = np.array([m.second.get(sourceKeys.instFluxErr) for m in matches])
if not np.all(np.isfinite(srcInstFluxErrArr)):
# this is an unpleasant hack; see DM-2308 requesting a better solution
self.log.warn("Source catalog does not have flux uncertainties; using sqrt(flux).")
srcInstFluxErrArr = np.sqrt(srcInstFluxArr)
# convert source instFlux from DN to an estimate of nJy
referenceFlux = (0*u.ABmag).to_value(u.nJy)
srcInstFluxArr = srcInstFluxArr * referenceFlux
srcInstFluxErrArr = srcInstFluxErrArr * referenceFlux
if not matches:
raise RuntimeError("No reference stars are available")
refSchema = matches[0].first.schema
applyColorTerms = self.config.applyColorTerms
applyCTReason = "config.applyColorTerms is %s" % (self.config.applyColorTerms,)
if self.config.applyColorTerms is None:
# apply color terms if color term data is available and photoCatName specified
ctDataAvail = len(self.config.colorterms.data) > 0
photoCatSpecified = self.config.photoCatName is not None
applyCTReason += " and data %s available" % ("is" if ctDataAvail else "is not")
applyCTReason += " and photoRefCat %s provided" % ("is" if photoCatSpecified else "is not")
applyColorTerms = ctDataAvail and photoCatSpecified
if applyColorTerms:
self.log.info("Applying color terms for filterName=%r, config.photoCatName=%s because %s",
filterName, self.config.photoCatName, applyCTReason)
colorterm = self.config.colorterms.getColorterm(
filterName=filterName, photoCatName=self.config.photoCatName, doRaise=True)
refCat = afwTable.SimpleCatalog(matches[0].first.schema)
# extract the matched refCat as a Catalog for the colorterm code
refCat.reserve(len(matches))
for x in matches:
record = refCat.addNew()
record.assign(x.first)
refMagArr, refMagErrArr = colorterm.getCorrectedMagnitudes(refCat, filterName)
fluxFieldList = [getRefFluxField(refSchema, filt) for filt in (colorterm.primary,
colorterm.secondary)]
else:
# no colorterms to apply
self.log.info("Not applying color terms because %s", applyCTReason)
colorterm = None
fluxFieldList = [getRefFluxField(refSchema, filterName)]
fluxField = getRefFluxField(refSchema, filterName)
fluxKey = refSchema.find(fluxField).key
refFluxArr = np.array([m.first.get(fluxKey) for m in matches])
try:
fluxErrKey = refSchema.find(fluxField + "Err").key
refFluxErrArr = np.array([m.first.get(fluxErrKey) for m in matches])
except KeyError:
# Reference catalogue may not have flux uncertainties; HACK DM-2308
self.log.warn("Reference catalog does not have flux uncertainties for %s; using sqrt(flux).",
fluxField)
refFluxErrArr = np.sqrt(refFluxArr)
refMagArr = u.Quantity(refFluxArr, u.nJy).to_value(u.ABmag)
# HACK convert to Jy until we have a replacement for this (DM-16903)
refMagErrArr = abMagErrFromFluxErr(refFluxErrArr*1e-9, refFluxArr*1e-9)
# compute the source catalog magnitudes and errors
srcMagArr = u.Quantity(srcInstFluxArr, u.nJy).to_value(u.ABmag)
# Fitting with error bars in both axes is hard
# for now ignore reference flux error, but ticket DM-2308 is a request for a better solution
# HACK convert to Jy until we have a replacement for this (DM-16903)
magErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
if self.config.magErrFloor != 0.0:
magErrArr = (magErrArr**2 + self.config.magErrFloor**2)**0.5
srcMagErrArr = abMagErrFromFluxErr(srcInstFluxErrArr*1e-9, srcInstFluxArr*1e-9)
good = np.isfinite(srcMagArr) & np.isfinite(refMagArr)
return pipeBase.Struct(
srcMag=srcMagArr[good],
refMag=refMagArr[good],
magErr=magErrArr[good],
srcMagErr=srcMagErrArr[good],
refMagErr=refMagErrArr[good],
refFluxFieldList=fluxFieldList,
)
def _formatCatalog(self, refCat, filterList):
"""Format a reference afw table into the final format.
This method applies reference selections and color terms as specified
by the config.
Parameters
----------
refCat : `lsst.afw.table.SourceCatalog`
Reference catalog in afw format.
filterList : `list` [`str`]
List of camera physicalFilter names to apply color terms.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
if self.config.doReferenceSelection:
goodSources = self.referenceSelector.selectSources(refCat)
selected = goodSources.selected
else:
selected = np.ones(len(refCat), dtype=bool)
npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'),
('dec', 'f8'),
('refMag', 'f4', (len(filterList), )),
('refMagErr', 'f4', (len(filterList), ))])
if npRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources.
return npRefCat
# Natively "coord_ra" and "coord_dec" are stored in radians.
# Doing this as an array rather than by row with the coord access is
# approximately 600x faster.
npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected])
npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected])
# Default (unset) values are 99.0
npRefCat['refMag'][:, :] = 99.0
npRefCat['refMagErr'][:, :] = 99.0
if self.config.doApplyColorTerms:
if isinstance(self.refObjLoader, ReferenceObjectLoader):
# Gen3
refCatName = self.refObjLoader.config.value.ref_dataset_name
else:
# Gen2
refCatName = self.refObjLoader.ref_dataset_name
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
if fluxField is None:
# There is no matching reference band.
# This will leave the column filled with 99s
continue
self.log.debug("Applying color terms for filterName='%s'", filterName)
colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True)
refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
# nan_to_num replaces nans with zeros, and this ensures
# that we select magnitudes that both filter out nans and are
# not very large (corresponding to very small fluxes), as "99"
# is a commen sentinel for illegal magnitudes.
good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected]) > 0.0))
npRefCat['refMag'][good, i] = refMag[selected][good]
npRefCat['refMagErr'][good, i] = refMagErr[selected][good]
else:
# No color terms to apply
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
# nan_to_num replaces nans with zeros, and this ensures that
# we select fluxes that both filter out nans and are positive.
good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0)
& (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0))
refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag)
refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good],
refCat[fluxField][selected][good])
npRefCat['refMag'][good, i] = refMag
npRefCat['refMagErr'][good, i] = refMagErr
return npRefCat
