def test_loadPixelBox(self):
"""Test the loadPixelBox routine."""
# This will create a box 50 degrees on a side.
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.pixelMargin = 0
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0),
dimensions=lsst.geom.Extent2I(1000, 1000))
crpix = lsst.geom.Point2D(500, 500)
crval = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
cdMatrix = afwGeom.makeCdMatrix(scale=0.05 * lsst.geom.degrees)
wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat
# This is a sanity check on the ranges; the exact selection depends
# on cos(dec) and the tangent-plane projection.
self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0)
self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0)
self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0)
self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0)
# The following is to ensure the reference catalog coords are
# getting corrected for proper motion when an epoch is provided.
# Use an extreme epoch so that differences in corrected coords
# will be significant. Note that this simply tests that the coords
# do indeed change when the epoch is passed. It makes no attempt
# at assessing the correctness of the change. This is left to the
# explicit testProperMotion() test below.
catWithEpoch = loader.loadPixelBox(bbox,
wcs,
'a',
bboxToSpherePadding=0,
epoch=astropy.time.Time(
30000,
format='mjd',
scale='tai')).refCat
self.assertFloatsNotEqual(cat['coord_ra'],
catWithEpoch['coord_ra'],
rtol=1.0e-4)
self.assertFloatsNotEqual(cat['coord_dec'],
catWithEpoch['coord_dec'],
rtol=1.0e-4)
class GenerateDonutCatalogOnlineTask(pipeBase.Task):
"""
Construct a source catalog from reference catalogs
and pointing information.
Parameters
----------
dataIds : `list`
List of the dataIds for the reference catalog shards.
refCats : `list`
List of the deferred dataset references pointing to the pieces
of the reference catalog we want in the butler.
**kwargs : dict[str, any]
Dictionary of input argument: new value for that input argument.
"""
ConfigClass = GenerateDonutCatalogOnlineTaskConfig
_DefaultName = "generateDonutCatalogOnlineTask"
def __init__(self, dataIds, refCats, **kwargs):
super().__init__(**kwargs)
refConfig = self.config.refObjLoader
# refObjLoader handles the interaction with the butler repository
# needed to get the pieces of the reference catalogs we need.
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
if self.config.doReferenceSelection:
self.makeSubtask("referenceSelector")
self.filterName = self.config.filterName
self.config.refObjLoader.pixelMargin = 0
self.config.refObjLoader.anyFilterMapsToThis = self.filterName
self.config.referenceSelector.magLimit.fluxField = f"{self.filterName}_flux"
self.config.referenceSelector.signalToNoise.fluxField = (
f"{self.filterName}_flux")
self.config.donutSelector.fluxField = f"{self.filterName}_flux"
if self.config.doDonutSelection:
self.makeSubtask("donutSelector")
@timeMethod
def run(self, detector, wcs):
"""Get the data from the reference catalog only from the
shards of the reference catalogs that overlap our pointing.
Parameters
----------
detector : `lsst.afw.cameraGeom.Detector`
Detector object from the camera.
wcs : `lsst.afw.geom.SkyWcs`
Wcs object defining the pixel to sky (and inverse) transform for
the supplied ``bbox``.
Returns
-------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- DonutCatalog: `pandas.DataFrame`
The final donut source catalog for the region.
"""
bbox = detector.getBBox()
# Get the refcatalog shard
skyBox = self.refObjLoader.loadPixelBox(bbox,
wcs,
filterName=self.filterName,
bboxToSpherePadding=0)
if not skyBox.refCat.isContiguous():
refCat = skyBox.refCat.copy(deep=True)
else:
refCat = skyBox.refCat
donutCatalog = self._formatCatalog(refCat, detector)
return pipeBase.Struct(donutCatalog=donutCatalog)
def _formatCatalog(self, refCat, detector):
"""Format a reference afw table into the final format.
Parameters
----------
refCat : `lsst.afw.table.SourceCatalog`
Reference catalog in afw format.
detector : `lsst.afw.cameraGeom.Detector`
Detector object from the camera.
Returns
-------
refCat : `pandas.DataFrame`
Reference catalog.
"""
if self.config.doReferenceSelection:
goodSources = self.referenceSelector.selectSources(refCat)
refSelected = goodSources.selected
else:
refSelected = np.ones(len(refCat), dtype=bool)
if self.config.doDonutSelection:
self.log.info("Running Donut Selector")
donutSelection = self.donutSelector.run(refCat, detector)
donutSelected = donutSelection.selected
else:
donutSelected = np.ones(len(refCat), dtype=bool)
selected = refSelected & donutSelected
npRefCat = np.zeros(
np.sum(selected),
dtype=[
("coord_ra", "f8"),
("coord_dec", "f8"),
("centroid_x", "f8"),
("centroid_y", "f8"),
("source_flux", "f8"),
],
)
if npRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources.
return npRefCat
# Natively "coord_ra" and "coord_dec" are stored in radians.
# Doing this as an array rather than by row with the coord access is
# approximately 600x faster.
npRefCat["coord_ra"] = refCat["coord_ra"][selected]
npRefCat["coord_dec"] = refCat["coord_dec"][selected]
npRefCat["centroid_x"] = refCat["centroid_x"][selected]
npRefCat["centroid_y"] = refCat["centroid_y"][selected]
fluxField = f"{self.filterName}_flux"
# nan_to_num replaces nans with zeros, and this ensures that
# we select fluxes that both filter out nans and are positive.
(good, ) = np.where(
(np.nan_to_num(refCat[fluxField][selected]) > 0.0)
& (np.nan_to_num(refCat[fluxField + "Err"][selected]) > 0.0))
npRefCat["source_flux"][good] = refCat[fluxField][selected][good]
return pd.DataFrame.from_records(npRefCat)
class LoadReferenceCatalogTask(pipeBase.Task):
"""Load multi-band reference objects from a reference catalog.
Parameters
----------
dataIds : iterable of `lsst.daf.butler.dataId`, optional
An iterable object of dataIds which point to reference catalogs
in a Gen3 repository. Required for Gen3.
refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`, optional
An iterable object of dataset refs for reference catalogs in
a Gen3 repository. Required for Gen3.
butler : `lsst.daf.persistence.Butler`, optional
A Gen2 butler. Required for Gen2.
"""
ConfigClass = LoadReferenceCatalogConfig
_DefaultName = "loadReferenceCatalog"
def __init__(self, dataIds=None, refCats=None, butler=None, **kwargs):
if dataIds is not None and refCats is not None:
pipeBase.Task.__init__(self, **kwargs)
refConfig = self.config.refObjLoader
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
elif butler is not None:
raise RuntimeError("LoadReferenceCatalogTask does not support Gen2 Butlers.")
else:
raise RuntimeError("Must instantiate LoadReferenceCatalogTask with "
"dataIds and refCats (Gen3)")
if self.config.doReferenceSelection:
self.makeSubtask('referenceSelector')
self._fluxFilters = None
self._fluxFields = None
self._referenceFilter = None
def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None,
bboxToSpherePadding=None):
"""Get a multi-band reference catalog by specifying a bounding box and WCS.
The catalog will be in `numpy.ndarray`, with positions proper-motion
corrected to "epoch" (if specified, and if the reference catalog has
proper motions); sources cut on a reference selector (if
"config.doReferenceSelection = True"); and color-terms applied (if
"config.doApplyColorTerms = True").
The format of the reference catalog will be of the format:
dtype = [('ra', 'np.float64'),
('dec', 'np.float64'),
('refMag', 'np.float32', (len(filterList), )),
('refMagErr', 'np.float32', (len(filterList), ))]
Reference magnitudes (AB) and errors will be 99 for non-detections
in a given band.
Parameters
----------
bbox : `lsst.geom.Box2I`
Box which bounds a region in pixel space.
wcs : `lsst.afw.geom.SkyWcs`
Wcs object defining the pixel to sky (and reverse) transform for
the supplied bbox.
filterList : `List` [ `str` ]
List of camera physicalFilter names to retrieve magnitudes.
epoch : `astropy.time.Time`, optional
Epoch to which to correct proper motion and parallax
(if available), or `None` to not apply such corrections.
bboxToSpherePadding : `int`, optional
Padding to account for translating a set of corners into a
spherical (convex) boundary that is certain to encompass the
entire area covered by the bbox.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
# Check if we have previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
center = wcs.pixelToSky(bbox.getCenter())
self._determineFluxFields(center, filterList)
skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter,
epoch=epoch,
bboxToSpherePadding=bboxToSpherePadding)
if not skyBox.refCat.isContiguous():
refCat = skyBox.refCat.copy(deep=True)
else:
refCat = skyBox.refCat
return self._formatCatalog(refCat, filterList)
def getSkyCircleCatalog(self, center, radius, filterList, epoch=None,
catalogFormat='numpy'):
"""Get a multi-band reference catalog by specifying a center and radius.
The catalog will be in `numpy.ndarray`, with positions proper-motion
corrected to "epoch" (if specified, and if the reference catalog has
proper motions); sources cut on a reference selector (if
"config.doReferenceSelection = True"); and color-terms applied (if
"config.doApplyColorTerms = True").
The format of the reference catalog will be of the format:
dtype = [('ra', 'np.float64'),
('dec', 'np.float64'),
('refMag', 'np.float32', (len(filterList), )),
('refMagErr', 'np.float32', (len(filterList), ))]
Reference magnitudes (AB) and errors will be 99 for non-detections
in a given band.
Parameters
----------
center : `lsst.geom.SpherePoint`
Point defining the center of the circular region.
radius : `lsst.geom.Angle`
Defines the angular radius of the circular region.
filterList : `List` [ `str` ]
List of camera physicalFilter names to retrieve magnitudes.
epoch : `astropy.time.Time`, optional
Epoch to which to correct proper motion and parallax
(if available), or `None` to not apply such corrections.
Parameters
----------
refCat : `numpy.ndarray`
Reference catalog.
"""
# Check if we have previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
self._determineFluxFields(center, filterList)
skyCircle = self.refObjLoader.loadSkyCircle(center, radius,
self._referenceFilter,
epoch=epoch)
if not skyCircle.refCat.isContiguous():
refCat = skyCircle.refCat.copy(deep=True)
else:
refCat = skyCircle.refCat
return self._formatCatalog(refCat, filterList)
def _formatCatalog(self, refCat, filterList):
"""Format a reference afw table into the final format.
This method applies reference selections and color terms as specified
by the config.
Parameters
----------
refCat : `lsst.afw.table.SourceCatalog`
Reference catalog in afw format.
filterList : `list` [`str`]
List of camera physicalFilter names to apply color terms.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
if self.config.doReferenceSelection:
goodSources = self.referenceSelector.selectSources(refCat)
selected = goodSources.selected
else:
selected = np.ones(len(refCat), dtype=bool)
npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'),
('dec', 'f8'),
('refMag', 'f4', (len(filterList), )),
('refMagErr', 'f4', (len(filterList), ))])
if npRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources.
return npRefCat
# Natively "coord_ra" and "coord_dec" are stored in radians.
# Doing this as an array rather than by row with the coord access is
# approximately 600x faster.
npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected])
npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected])
# Default (unset) values are 99.0
npRefCat['refMag'][:, :] = 99.0
npRefCat['refMagErr'][:, :] = 99.0
if self.config.doApplyColorTerms:
if isinstance(self.refObjLoader, ReferenceObjectLoader):
# Gen3
refCatName = self.refObjLoader.config.value.ref_dataset_name
else:
# Gen2
refCatName = self.refObjLoader.ref_dataset_name
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
if fluxField is None:
# There is no matching reference band.
# This will leave the column filled with 99s
continue
self.log.debug("Applying color terms for filterName='%s'", filterName)
colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True)
refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
# nan_to_num replaces nans with zeros, and this ensures
# that we select magnitudes that both filter out nans and are
# not very large (corresponding to very small fluxes), as "99"
# is a commen sentinel for illegal magnitudes.
good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected]) > 0.0))
npRefCat['refMag'][good, i] = refMag[selected][good]
npRefCat['refMagErr'][good, i] = refMagErr[selected][good]
else:
# No color terms to apply
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
# nan_to_num replaces nans with zeros, and this ensures that
# we select fluxes that both filter out nans and are positive.
good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0)
& (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0))
refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag)
refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good],
refCat[fluxField][selected][good])
npRefCat['refMag'][good, i] = refMag
npRefCat['refMagErr'][good, i] = refMagErr
return npRefCat
def _determineFluxFields(self, center, filterList):
"""Determine the flux field names for a reference catalog.
This method sets self._fluxFields, self._referenceFilter.
Parameters
----------
center : `lsst.geom.SpherePoint`
The center around which to load test sources.
filterList : `list` [`str`]
List of camera physicalFilter names.
"""
# Search for a good filter to use to load the reference catalog
# via the refObjLoader task which requires a valid filterName
foundReferenceFilter = False
# Store the original config
_config = self.refObjLoader.config
configTemp = LoadReferenceObjectsConfig()
configIntersection = {k: getattr(self.refObjLoader.config, k)
for k, v in self.refObjLoader.config.toDict().items()
if (k in configTemp.keys() and k != "connections")}
# We must turn off the proper motion checking to find the refFilter.
configIntersection['requireProperMotion'] = False
configTemp.update(**configIntersection)
self.refObjLoader.config = configTemp
for filterName in filterList:
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is None:
continue
try:
results = self.refObjLoader.loadSkyCircle(center,
0.05*lsst.geom.degrees,
refFilterName)
foundReferenceFilter = True
self._referenceFilter = refFilterName
break
except RuntimeError as err:
# This just means that the filterName wasn't listed
# in the reference catalog. This is okay.
if 'not find flux' in err.args[0]:
# The filterName wasn't listed in the reference catalog.
# This is not a fatal failure (yet)
pass
else:
raise err
self.refObjLoader.config = _config
if not foundReferenceFilter:
raise RuntimeError("Could not find any valid flux field(s) %s" %
(", ".join(filterList)))
# Record self._fluxFilters for checks on subsequent calls
self._fluxFilters = filterList
# Retrieve all the fluxField names
self._fluxFields = []
for filterName in filterList:
fluxField = None
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is not None:
try:
fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName)
except RuntimeError:
# This flux field isn't available. Set to None
fluxField = None
if fluxField is None:
self.log.warning('No reference flux field for camera filter %s', filterName)
self._fluxFields.append(fluxField)