def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if self.config.doAstrometry:
refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.astromRefCat],
refCats=inputs.pop('astromRefCat'),
config=self.config.astromRefObjLoader, log=self.log)
self.pixelMargin = refObjLoader.config.pixelMargin
self.astrometry.setRefObjLoader(refObjLoader)
if self.config.doPhotoCal:
photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.photoRefCat],
refCats=inputs.pop('photoRefCat'),
config=self.config.photoRefObjLoader,
log=self.log)
self.pixelMargin = photoRefObjLoader.config.pixelMargin
self.photoCal.match.setRefObjLoader(photoRefObjLoader)
outputs = self.run(**inputs)
if self.config.doWriteMatches and self.config.doAstrometry:
normalizedMatches = afwTable.packMatches(outputs.astromMatches)
normalizedMatches.table.setMetadata(outputs.matchMeta)
if self.config.doWriteMatchesDenormalized:
denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
outputs.matchesDenormalized = denormMatches
outputs.matches = normalizedMatches
butlerQC.put(outputs, outputRefs)
def test_loadSkyCircle(self):
"""Test the loadSkyCircle routine."""
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs], self.handles)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
cat = loader.loadSkyCircle(
center,
30.0 * lsst.geom.degrees,
filterName='a',
).refCat
# Check that the max distance is less than the radius
dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']),
np.rad2deg(cat['coord_dec']))
self.assertLess(np.max(dist), 30.0)
# Check that all the objects from the two catalogs are here.
dist = sphdist(180.0, 0.0, self.skyCatalog['ra_icrs'],
self.skyCatalog['dec_icrs'])
inside, = (dist < 30.0).nonzero()
self.assertEqual(len(cat), len(inside))
self.assertTrue(cat.isContiguous())
self.assertEqual(len(np.unique(cat['id'])), len(cat))
# A default-loaded sky circle should not have centroids
self.assertNotIn('centroid_x', cat.schema)
self.assertNotIn('centroid_y', cat.schema)
self.assertNotIn('hasCentroid', cat.schema)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
expId, expBits = butler.registry.packDataId("visit_detector",
inputDataIds['exposure'],
returnMaxBits=True)
inputData['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if self.config.doAstrometry:
refObjLoader = ReferenceObjectLoader(dataIds=inputDataIds['astromRefCat'],
butler=butler,
config=self.config.astromRefObjLoader,
log=self.log)
self.pixelMargin = refObjLoader.config.pixelMargin
self.astrometry.setRefObjLoader(refObjLoader)
if self.config.doPhotoCal:
photoRefObjLoader = ReferenceObjectLoader(inputDataIds['photoRefCat'],
butler,
self.config.photoRefObjLoader,
self.log)
self.pixelMargin = photoRefObjLoader.config.pixelMargin
self.photoCal.match.setRefObjLoader(photoRefObjLoader)
results = self.run(**inputData)
if self.config.doWriteMatches:
normalizedMatches = afwTable.packMatches(results.astromMatches)
normalizedMatches.table.setMetadata(results.matchMeta)
if self.config.doWriteMatchesDenormalized:
denormMatches = denormalizeMatches(results.astromMatches, results.matchMeta)
results.matchesDenormalized = denormMatches
results.matches = normalizedMatches
return results
def test_loadPixelBox(self):
"""Test the loadPixelBox routine."""
# This will create a box 50 degrees on a side.
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.pixelMargin = 0
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0),
dimensions=lsst.geom.Extent2I(1000, 1000))
crpix = lsst.geom.Point2D(500, 500)
crval = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
cdMatrix = afwGeom.makeCdMatrix(scale=0.05 * lsst.geom.degrees)
wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat
# This is a sanity check on the ranges; the exact selection depends
# on cos(dec) and the tangent-plane projection.
self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0)
self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0)
self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0)
self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0)
# The following is to ensure the reference catalog coords are
# getting corrected for proper motion when an epoch is provided.
# Use an extreme epoch so that differences in corrected coords
# will be significant. Note that this simply tests that the coords
# do indeed change when the epoch is passed. It makes no attempt
# at assessing the correctness of the change. This is left to the
# explicit testProperMotion() test below.
catWithEpoch = loader.loadPixelBox(bbox,
wcs,
'a',
bboxToSpherePadding=0,
epoch=astropy.time.Time(
30000,
format='mjd',
scale='tai')).refCat
self.assertFloatsNotEqual(cat['coord_ra'],
catWithEpoch['coord_ra'],
rtol=1.0e-4)
self.assertFloatsNotEqual(cat['coord_dec'],
catWithEpoch['coord_dec'],
rtol=1.0e-4)
def getRefObjLoader(self, refCatalogList):
"""
Create a `ReferenceObjectLoader` from available reference catalogs
in the repository.
Parameters
----------
refCatalogList : `list`
List of deferred butler references for the reference catalogs.
Returns
-------
`lsst.meas.algorithms.ReferenceObjectsLoader`
Object to conduct spatial searches through the reference catalogs
"""
refObjLoader = ReferenceObjectLoader(
dataIds=[ref.dataId for ref in refCatalogList],
refCats=refCatalogList,
config=LoadReferenceObjectsConfig(),
)
# This removes the padding around the border of detector BBox when
# matching to reference catalog.
# We remove this since we only want sources within detector.
refObjLoader.config.pixelMargin = 0
return refObjLoader
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.astromRefCat],
refCats=inputs.pop('astromRefCat'),
config=self.config.astromRefObjLoader,
log=self.log)
refObjLoader.pixelMargin = 1000
self.astrometry.setRefObjLoader(refObjLoader)
# See L603 (def runQuantum(self, butlerQC, inputRefs, outputRefs):)
# in calibrate.py to put photocal back in
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def test_requireProperMotion(self):
"""Tests of the requireProperMotion config field."""
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.requireProperMotion = True
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
# Test that we require an epoch set.
msg = 'requireProperMotion=True but epoch not provided to loader'
with self.assertRaisesRegex(RuntimeError, msg):
loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a')
def test_filterMap(self):
"""Test filterMap parameters."""
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.filterMap = {'aprime': 'a'}
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
result = loader.loadSkyCircle(
center,
30.0 * lsst.geom.degrees,
filterName='aprime',
)
self.assertEqual(result.fluxField, 'aprime_camFlux')
self.assertFloatsEqual(result.refCat['aprime_camFlux'],
result.refCat['a_flux'])
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
inputs['dataId'] = str(butlerQC.quantum.dataId)
refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.refCat],
refCats=inputs.pop("refCat"),
config=self.config.refObjLoader)
output = self.run(**inputs, refObjLoader=refObjLoader)
butlerQC.put(output, outputRefs)
def __init__(self, dataIds=None, refCats=None, butler=None, **kwargs):
if dataIds is not None and refCats is not None:
pipeBase.Task.__init__(self, **kwargs)
refConfig = self.config.refObjLoader
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
elif butler is not None:
raise RuntimeError("LoadReferenceCatalogTask does not support Gen2 Butlers.")
else:
raise RuntimeError("Must instantiate LoadReferenceCatalogTask with "
"dataIds and refCats (Gen3)")
if self.config.doReferenceSelection:
self.makeSubtask('referenceSelector')
self._fluxFilters = None
self._fluxFields = None
self._referenceFilter = None
def __init__(self, dataIds, refCats, **kwargs):
super().__init__(**kwargs)
refConfig = self.config.refObjLoader
# refObjLoader handles the interaction with the butler repository
# needed to get the pieces of the reference catalogs we need.
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
if self.config.doReferenceSelection:
self.makeSubtask("referenceSelector")
self.filterName = self.config.filterName
self.config.refObjLoader.pixelMargin = 0
self.config.refObjLoader.anyFilterMapsToThis = self.filterName
self.config.referenceSelector.magLimit.fluxField = f"{self.filterName}_flux"
self.config.referenceSelector.signalToNoise.fluxField = (
f"{self.filterName}_flux")
self.config.donutSelector.fluxField = f"{self.filterName}_flux"
if self.config.doDonutSelection:
self.makeSubtask("donutSelector")
def test_properMotion(self):
"""Test proper motion correction."""
loaderConfig = ReferenceObjectLoader.ConfigClass()
loaderConfig.filterMap = {'aprime': 'a'}
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in self.datasetRefs],
self.handles,
config=loaderConfig)
center = lsst.geom.SpherePoint(180.0 * lsst.geom.degrees,
0.0 * lsst.geom.degrees)
cat = loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a').refCat
# Zero epoch change --> no proper motion correction (except minor numerical effects)
cat_pm = loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a',
epoch=self.epoch).refCat
self.assertFloatsAlmostEqual(cat_pm['coord_ra'],
cat['coord_ra'],
rtol=1.0e-14)
self.assertFloatsAlmostEqual(cat_pm['coord_dec'],
cat['coord_dec'],
rtol=1.0e-14)
self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr'])
self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr'])
# One year difference
cat_pm = loader.loadSkyCircle(center,
30.0 * lsst.geom.degrees,
filterName='a',
epoch=self.epoch +
1.0 * astropy.units.yr).refCat
self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
for orig, ref in zip(cat, cat_pm):
self.assertAnglesAlmostEqual(orig.getCoord().separation(
ref.getCoord()),
self.properMotionAmt,
maxDiff=1.0e-6 * lsst.geom.arcseconds)
self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(
ref.getCoord()),
self.properMotionDir,
maxDiff=1.0e-4 * lsst.geom.arcseconds)
predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
def test_fgcmLoadReference(self):
"""
Test loading of the fgcm reference catalogs.
"""
filterList = ['HSC-R', 'HSC-I']
config = fgcmcal.FgcmLoadReferenceCatalogConfig()
config.applyColorTerms = True
config.filterMap = {'HSC-R': 'r', 'HSC-I': 'i'}
config.colorterms.data = {}
config.colorterms.data[
'ps1*'] = lsst.pipe.tasks.colorterms.ColortermDict()
config.colorterms.data['ps1*'].data = {}
config.colorterms.data['ps1*'].data[
'HSC-R'] = lsst.pipe.tasks.colorterms.Colorterm()
config.colorterms.data['ps1*'].data['HSC-R'].primary = 'r'
config.colorterms.data['ps1*'].data['HSC-R'].secondary = 'i'
config.colorterms.data['ps1*'].data['HSC-R'].c0 = -0.000144
config.colorterms.data['ps1*'].data['HSC-R'].c1 = 0.001369
config.colorterms.data['ps1*'].data['HSC-R'].c2 = -0.008380
config.colorterms.data['ps1*'].data[
'HSC-I'] = lsst.pipe.tasks.colorterms.Colorterm()
config.colorterms.data['ps1*'].data['HSC-I'].primary = 'i'
config.colorterms.data['ps1*'].data['HSC-I'].secondary = 'z'
config.colorterms.data['ps1*'].data['HSC-I'].c0 = 0.000643
config.colorterms.data['ps1*'].data['HSC-I'].c1 = -0.130078
config.colorterms.data['ps1*'].data['HSC-I'].c2 = -0.006855
refCatName = 'ps1_pv3_3pi_20170110'
butler = lsst.daf.butler.Butler(
self.repo,
instrument='HSC',
collections=['HSC/testdata', 'refcats/gen2'])
refs = set(butler.registry.queryDatasets(refCatName))
dataIds = [butler.registry.expandDataId(ref.dataId) for ref in refs]
refCats = [butler.getDirectDeferred(ref) for ref in refs]
refConfig = LoadReferenceObjectsConfig()
refConfig.filterMap = config.filterMap
refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig)
loadCat = fgcmcal.FgcmLoadReferenceCatalogTask(
refObjLoader=refObjLoader, refCatName=refCatName, config=config)
ra = 337.656174
dec = 0.823595
rad = 0.1
refCat = loadCat.getFgcmReferenceStarsSkyCircle(
ra, dec, rad, filterList)
# Check the number of mags and ranges
self.assertEqual(len(filterList), refCat['refMag'].shape[1])
self.assertEqual(len(filterList), refCat['refMagErr'].shape[1])
self.assertLess(np.max(refCat['refMag'][:, 0]), 99.1)
self.assertLess(np.max(refCat['refMag'][:, 1]), 99.1)
self.assertLess(np.max(refCat['refMagErr'][:, 0]), 99.1)
self.assertLess(np.max(refCat['refMagErr'][:, 1]), 99.1)
test, = np.where((refCat['refMag'][:, 0] < 30.0)
& (refCat['refMag'][:, 1] < 30.0))
self.assertGreater(test.size, 0)
# Check the separations from the center
self.assertLess(
np.max(esutil.coords.sphdist(ra, dec, refCat['ra'],
refCat['dec'])), rad)
# And load a healpixel
nside = 256
pixel = 387520
refCat = loadCat.getFgcmReferenceStarsHealpix(nside, pixel, filterList)
ipring = hp.ang2pix(nside, np.radians(90.0 - refCat['dec']),
np.radians(refCat['ra']))
self.assertEqual(pixel, np.max(ipring))
self.assertEqual(pixel, np.min(ipring))
class LoadReferenceCatalogTask(pipeBase.Task):
"""Load multi-band reference objects from a reference catalog.
Parameters
----------
dataIds : iterable of `lsst.daf.butler.dataId`, optional
An iterable object of dataIds which point to reference catalogs
in a Gen3 repository. Required for Gen3.
refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`, optional
An iterable object of dataset refs for reference catalogs in
a Gen3 repository. Required for Gen3.
butler : `lsst.daf.persistence.Butler`, optional
A Gen2 butler. Required for Gen2.
"""
ConfigClass = LoadReferenceCatalogConfig
_DefaultName = "loadReferenceCatalog"
def __init__(self, dataIds=None, refCats=None, butler=None, **kwargs):
if dataIds is not None and refCats is not None:
pipeBase.Task.__init__(self, **kwargs)
refConfig = self.config.refObjLoader
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
elif butler is not None:
raise RuntimeError("LoadReferenceCatalogTask does not support Gen2 Butlers.")
else:
raise RuntimeError("Must instantiate LoadReferenceCatalogTask with "
"dataIds and refCats (Gen3)")
if self.config.doReferenceSelection:
self.makeSubtask('referenceSelector')
self._fluxFilters = None
self._fluxFields = None
self._referenceFilter = None
def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None,
bboxToSpherePadding=None):
"""Get a multi-band reference catalog by specifying a bounding box and WCS.
The catalog will be in `numpy.ndarray`, with positions proper-motion
corrected to "epoch" (if specified, and if the reference catalog has
proper motions); sources cut on a reference selector (if
"config.doReferenceSelection = True"); and color-terms applied (if
"config.doApplyColorTerms = True").
The format of the reference catalog will be of the format:
dtype = [('ra', 'np.float64'),
('dec', 'np.float64'),
('refMag', 'np.float32', (len(filterList), )),
('refMagErr', 'np.float32', (len(filterList), ))]
Reference magnitudes (AB) and errors will be 99 for non-detections
in a given band.
Parameters
----------
bbox : `lsst.geom.Box2I`
Box which bounds a region in pixel space.
wcs : `lsst.afw.geom.SkyWcs`
Wcs object defining the pixel to sky (and reverse) transform for
the supplied bbox.
filterList : `List` [ `str` ]
List of camera physicalFilter names to retrieve magnitudes.
epoch : `astropy.time.Time`, optional
Epoch to which to correct proper motion and parallax
(if available), or `None` to not apply such corrections.
bboxToSpherePadding : `int`, optional
Padding to account for translating a set of corners into a
spherical (convex) boundary that is certain to encompass the
entire area covered by the bbox.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
# Check if we have previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
center = wcs.pixelToSky(bbox.getCenter())
self._determineFluxFields(center, filterList)
skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter,
epoch=epoch,
bboxToSpherePadding=bboxToSpherePadding)
if not skyBox.refCat.isContiguous():
refCat = skyBox.refCat.copy(deep=True)
else:
refCat = skyBox.refCat
return self._formatCatalog(refCat, filterList)
def getSkyCircleCatalog(self, center, radius, filterList, epoch=None,
catalogFormat='numpy'):
"""Get a multi-band reference catalog by specifying a center and radius.
The catalog will be in `numpy.ndarray`, with positions proper-motion
corrected to "epoch" (if specified, and if the reference catalog has
proper motions); sources cut on a reference selector (if
"config.doReferenceSelection = True"); and color-terms applied (if
"config.doApplyColorTerms = True").
The format of the reference catalog will be of the format:
dtype = [('ra', 'np.float64'),
('dec', 'np.float64'),
('refMag', 'np.float32', (len(filterList), )),
('refMagErr', 'np.float32', (len(filterList), ))]
Reference magnitudes (AB) and errors will be 99 for non-detections
in a given band.
Parameters
----------
center : `lsst.geom.SpherePoint`
Point defining the center of the circular region.
radius : `lsst.geom.Angle`
Defines the angular radius of the circular region.
filterList : `List` [ `str` ]
List of camera physicalFilter names to retrieve magnitudes.
epoch : `astropy.time.Time`, optional
Epoch to which to correct proper motion and parallax
(if available), or `None` to not apply such corrections.
Parameters
----------
refCat : `numpy.ndarray`
Reference catalog.
"""
# Check if we have previously cached values for the fluxFields
if self._fluxFilters is None or self._fluxFilters != filterList:
self._determineFluxFields(center, filterList)
skyCircle = self.refObjLoader.loadSkyCircle(center, radius,
self._referenceFilter,
epoch=epoch)
if not skyCircle.refCat.isContiguous():
refCat = skyCircle.refCat.copy(deep=True)
else:
refCat = skyCircle.refCat
return self._formatCatalog(refCat, filterList)
def _formatCatalog(self, refCat, filterList):
"""Format a reference afw table into the final format.
This method applies reference selections and color terms as specified
by the config.
Parameters
----------
refCat : `lsst.afw.table.SourceCatalog`
Reference catalog in afw format.
filterList : `list` [`str`]
List of camera physicalFilter names to apply color terms.
Returns
-------
refCat : `numpy.ndarray`
Reference catalog.
"""
if self.config.doReferenceSelection:
goodSources = self.referenceSelector.selectSources(refCat)
selected = goodSources.selected
else:
selected = np.ones(len(refCat), dtype=bool)
npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'),
('dec', 'f8'),
('refMag', 'f4', (len(filterList), )),
('refMagErr', 'f4', (len(filterList), ))])
if npRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources.
return npRefCat
# Natively "coord_ra" and "coord_dec" are stored in radians.
# Doing this as an array rather than by row with the coord access is
# approximately 600x faster.
npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected])
npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected])
# Default (unset) values are 99.0
npRefCat['refMag'][:, :] = 99.0
npRefCat['refMagErr'][:, :] = 99.0
if self.config.doApplyColorTerms:
if isinstance(self.refObjLoader, ReferenceObjectLoader):
# Gen3
refCatName = self.refObjLoader.config.value.ref_dataset_name
else:
# Gen2
refCatName = self.refObjLoader.ref_dataset_name
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
if fluxField is None:
# There is no matching reference band.
# This will leave the column filled with 99s
continue
self.log.debug("Applying color terms for filterName='%s'", filterName)
colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True)
refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
# nan_to_num replaces nans with zeros, and this ensures
# that we select magnitudes that both filter out nans and are
# not very large (corresponding to very small fluxes), as "99"
# is a commen sentinel for illegal magnitudes.
good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0)
& (np.nan_to_num(refMagErr[selected]) > 0.0))
npRefCat['refMag'][good, i] = refMag[selected][good]
npRefCat['refMagErr'][good, i] = refMagErr[selected][good]
else:
# No color terms to apply
for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
# nan_to_num replaces nans with zeros, and this ensures that
# we select fluxes that both filter out nans and are positive.
good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0)
& (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0))
refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag)
refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good],
refCat[fluxField][selected][good])
npRefCat['refMag'][good, i] = refMag
npRefCat['refMagErr'][good, i] = refMagErr
return npRefCat
def _determineFluxFields(self, center, filterList):
"""Determine the flux field names for a reference catalog.
This method sets self._fluxFields, self._referenceFilter.
Parameters
----------
center : `lsst.geom.SpherePoint`
The center around which to load test sources.
filterList : `list` [`str`]
List of camera physicalFilter names.
"""
# Search for a good filter to use to load the reference catalog
# via the refObjLoader task which requires a valid filterName
foundReferenceFilter = False
# Store the original config
_config = self.refObjLoader.config
configTemp = LoadReferenceObjectsConfig()
configIntersection = {k: getattr(self.refObjLoader.config, k)
for k, v in self.refObjLoader.config.toDict().items()
if (k in configTemp.keys() and k != "connections")}
# We must turn off the proper motion checking to find the refFilter.
configIntersection['requireProperMotion'] = False
configTemp.update(**configIntersection)
self.refObjLoader.config = configTemp
for filterName in filterList:
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is None:
continue
try:
results = self.refObjLoader.loadSkyCircle(center,
0.05*lsst.geom.degrees,
refFilterName)
foundReferenceFilter = True
self._referenceFilter = refFilterName
break
except RuntimeError as err:
# This just means that the filterName wasn't listed
# in the reference catalog. This is okay.
if 'not find flux' in err.args[0]:
# The filterName wasn't listed in the reference catalog.
# This is not a fatal failure (yet)
pass
else:
raise err
self.refObjLoader.config = _config
if not foundReferenceFilter:
raise RuntimeError("Could not find any valid flux field(s) %s" %
(", ".join(filterList)))
# Record self._fluxFilters for checks on subsequent calls
self._fluxFilters = filterList
# Retrieve all the fluxField names
self._fluxFields = []
for filterName in filterList:
fluxField = None
if self.config.refObjLoader.anyFilterMapsToThis is not None:
refFilterName = self.config.refObjLoader.anyFilterMapsToThis
else:
refFilterName = self.config.refObjLoader.filterMap.get(filterName)
if refFilterName is not None:
try:
fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName)
except RuntimeError:
# This flux field isn't available. Set to None
fluxField = None
if fluxField is None:
self.log.warning('No reference flux field for camera filter %s', filterName)
self._fluxFields.append(fluxField)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputRefDict = butlerQC.get(inputRefs)
sourceTableHandles = inputRefDict['sourceTable_visit']
self.log.info("Running with %d sourceTable_visit handles",
len(sourceTableHandles))
sourceTableHandleDict = {
sourceTableHandle.dataId['visit']: sourceTableHandle
for sourceTableHandle in sourceTableHandles
}
if self.config.doReferenceMatches:
# Get the LUT handle
lutHandle = inputRefDict['fgcmLookUpTable']
# Prepare the reference catalog loader
refConfig = LoadReferenceObjectsConfig()
refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap
refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
log=self.log,
config=refConfig)
self.makeSubtask('fgcmLoadReferenceCatalog',
refObjLoader=refObjLoader,
refCatName=self.config.connections.refCat)
else:
lutHandle = None
# Compute aperture radius if necessary. This is useful to do now before
# any heave lifting has happened (fail early).
calibFluxApertureRadius = None
if self.config.doSubtractLocalBackground:
try:
calibFluxApertureRadius = computeApertureRadiusFromName(
self.config.instFluxField)
except RuntimeError as e:
raise RuntimeError(
"Could not determine aperture radius from %s. "
"Cannot use doSubtractLocalBackground." %
(self.config.instFluxField)) from e
visitSummaryHandles = inputRefDict['visitSummary']
visitSummaryHandleDict = {
visitSummaryHandle.dataId['visit']: visitSummaryHandle
for visitSummaryHandle in visitSummaryHandles
}
camera = inputRefDict['camera']
groupedHandles = self._groupHandles(sourceTableHandleDict,
visitSummaryHandleDict)
if self.config.doModelErrorsWithBackground:
bkgHandles = inputRefDict['background']
bkgHandleDict = {(bkgHandle.dataId.byName()['visit'],
bkgHandle.dataId.byName()['detector']): bkgHandle
for bkgHandle in bkgHandles}
else:
bkgHandleDict = None
visitCat = self.fgcmMakeVisitCatalog(camera,
groupedHandles,
bkgHandleDict=bkgHandleDict)
rad = calibFluxApertureRadius
fgcmStarObservationCat = self.fgcmMakeAllStarObservations(
groupedHandles,
visitCat,
self.sourceSchema,
camera,
calibFluxApertureRadius=rad)
butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog)
butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations)
fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(
visitCat, fgcmStarObservationCat, lutHandle=lutHandle)
butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds)
butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices)
if fgcmRefCat is not None:
butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = {}
handleDict['camera'] = butlerQC.get(inputRefs.camera)
handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
handleDict['fgcmVisitCatalog'] = butlerQC.get(
inputRefs.fgcmVisitCatalog)
handleDict['fgcmStandardStars'] = butlerQC.get(
inputRefs.fgcmStandardStars)
if self.config.doZeropointOutput:
handleDict['fgcmZeropoints'] = butlerQC.get(
inputRefs.fgcmZeropoints)
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
if self.config.doAtmosphereOutput:
handleDict['fgcmAtmosphereParameters'] = butlerQC.get(
inputRefs.fgcmAtmosphereParameters)
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
if self.config.doReferenceCalibration:
refConfig = LoadReferenceObjectsConfig()
self.refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
log=self.log,
config=refConfig)
else:
self.refObjLoader = None
struct = self.run(handleDict, self.config.physicalFilterMap)
# Output the photoCalib exposure catalogs
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
# Output the atmospheres
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
if self.config.doReferenceCalibration:
# Turn offset into simple catalog for persistence if necessary
schema = afwTable.Schema()
schema.addField('offset',
type=np.float64,
doc="Post-process calibration offset (mag)")
offsetCat = afwTable.BaseCatalog(schema)
offsetCat.resize(len(struct.offsets))
offsetCat['offset'][:] = struct.offsets
butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
return
class FgcmOutputProductsTask(pipeBase.PipelineTask):
"""
Output products from FGCM global calibration.
"""
ConfigClass = FgcmOutputProductsConfig
_DefaultName = "fgcmOutputProducts"
def __init__(self, **kwargs):
super().__init__(**kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = {}
handleDict['camera'] = butlerQC.get(inputRefs.camera)
handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
handleDict['fgcmVisitCatalog'] = butlerQC.get(
inputRefs.fgcmVisitCatalog)
handleDict['fgcmStandardStars'] = butlerQC.get(
inputRefs.fgcmStandardStars)
if self.config.doZeropointOutput:
handleDict['fgcmZeropoints'] = butlerQC.get(
inputRefs.fgcmZeropoints)
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
if self.config.doAtmosphereOutput:
handleDict['fgcmAtmosphereParameters'] = butlerQC.get(
inputRefs.fgcmAtmosphereParameters)
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
if self.config.doReferenceCalibration:
refConfig = LoadReferenceObjectsConfig()
self.refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
log=self.log,
config=refConfig)
else:
self.refObjLoader = None
struct = self.run(handleDict, self.config.physicalFilterMap)
# Output the photoCalib exposure catalogs
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
# Output the atmospheres
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
if self.config.doReferenceCalibration:
# Turn offset into simple catalog for persistence if necessary
schema = afwTable.Schema()
schema.addField('offset',
type=np.float64,
doc="Post-process calibration offset (mag)")
offsetCat = afwTable.BaseCatalog(schema)
offsetCat.resize(len(struct.offsets))
offsetCat['offset'][:] = struct.offsets
butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
return
def run(self, handleDict, physicalFilterMap):
"""Run the output products task.
Parameters
----------
handleDict : `dict`
All handles are `lsst.daf.butler.DeferredDatasetHandle`
handle dictionary with keys:
``"camera"``
Camera object (`lsst.afw.cameraGeom.Camera`)
``"fgcmLookUpTable"``
handle for the FGCM look-up table.
``"fgcmVisitCatalog"``
handle for visit summary catalog.
``"fgcmStandardStars"``
handle for the output standard star catalog.
``"fgcmZeropoints"``
handle for the zeropoint data catalog.
``"fgcmAtmosphereParameters"``
handle for the atmosphere parameter catalog.
``"fgcmBuildStarsTableConfig"``
Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`.
physicalFilterMap : `dict`
Dictionary of mappings from physical filter to FGCM band.
Returns
-------
retStruct : `lsst.pipe.base.Struct`
Output structure with keys:
offsets : `np.ndarray`
Final reference offsets, per band.
atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
Generator that returns (visit, transmissionCurve) tuples.
photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
Generator that returns (visit, exposureCatalog) tuples.
"""
stdCat = handleDict['fgcmStandardStars'].get()
md = stdCat.getMetadata()
bands = md.getArray('BANDS')
if self.config.doReferenceCalibration:
lutCat = handleDict['fgcmLookUpTable'].get()
offsets = self._computeReferenceOffsets(stdCat, lutCat,
physicalFilterMap, bands)
else:
offsets = np.zeros(len(bands))
del stdCat
if self.config.doZeropointOutput:
zptCat = handleDict['fgcmZeropoints'].get()
visitCat = handleDict['fgcmVisitCatalog'].get()
pcgen = self._outputZeropoints(handleDict['camera'], zptCat,
visitCat, offsets, bands,
physicalFilterMap)
else:
pcgen = None
if self.config.doAtmosphereOutput:
atmCat = handleDict['fgcmAtmosphereParameters'].get()
atmgen = self._outputAtmospheres(handleDict, atmCat)
else:
atmgen = None
retStruct = pipeBase.Struct(offsets=offsets, atmospheres=atmgen)
retStruct.photoCalibCatalogs = pcgen
return retStruct
def generateTractOutputProducts(self, handleDict, tract, visitCat, zptCat,
atmCat, stdCat, fgcmBuildStarsConfig):
"""
Generate the output products for a given tract, as specified in the config.
This method is here to have an alternate entry-point for
FgcmCalibrateTract.
Parameters
----------
handleDict : `dict`
All handles are `lsst.daf.butler.DeferredDatasetHandle`
handle dictionary with keys:
``"camera"``
Camera object (`lsst.afw.cameraGeom.Camera`)
``"fgcmLookUpTable"``
handle for the FGCM look-up table.
tract : `int`
Tract number
visitCat : `lsst.afw.table.BaseCatalog`
FGCM visitCat from `FgcmBuildStarsTask`
zptCat : `lsst.afw.table.BaseCatalog`
FGCM zeropoint catalog from `FgcmFitCycleTask`
atmCat : `lsst.afw.table.BaseCatalog`
FGCM atmosphere parameter catalog from `FgcmFitCycleTask`
stdCat : `lsst.afw.table.SimpleCatalog`
FGCM standard star catalog from `FgcmFitCycleTask`
fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig`
Configuration object from `FgcmBuildStarsTask`
Returns
-------
retStruct : `lsst.pipe.base.Struct`
Output structure with keys:
offsets : `np.ndarray`
Final reference offsets, per band.
atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
Generator that returns (visit, transmissionCurve) tuples.
photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
Generator that returns (visit, exposureCatalog) tuples.
"""
physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap
md = stdCat.getMetadata()
bands = md.getArray('BANDS')
if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian:
raise RuntimeError(
"Cannot compose the WCS jacobian if it hasn't been applied "
"in fgcmBuildStarsTask.")
if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian:
self.log.warning(
"Jacobian was applied in build-stars but doComposeWcsJacobian is not set."
)
if self.config.doReferenceCalibration:
lutCat = handleDict['fgcmLookUpTable'].get()
offsets = self._computeReferenceOffsets(stdCat, lutCat, bands,
physicalFilterMap)
else:
offsets = np.zeros(len(bands))
if self.config.doZeropointOutput:
pcgen = self._outputZeropoints(handleDict['camera'], zptCat,
visitCat, offsets, bands,
physicalFilterMap)
else:
pcgen = None
if self.config.doAtmosphereOutput:
atmgen = self._outputAtmospheres(handleDict, atmCat)
else:
atmgen = None
retStruct = pipeBase.Struct(offsets=offsets, atmospheres=atmgen)
retStruct.photoCalibCatalogs = pcgen
return retStruct
def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap,
bands):
"""
Compute offsets relative to a reference catalog.
This method splits the star catalog into healpix pixels
and computes the calibration transfer for a sample of
these pixels to approximate the 'absolute' calibration
values (on for each band) to apply to transfer the
absolute scale.
Parameters
----------
stdCat : `lsst.afw.table.SimpleCatalog`
FGCM standard stars
lutCat : `lsst.afw.table.SimpleCatalog`
FGCM Look-up table
physicalFilterMap : `dict`
Dictionary of mappings from physical filter to FGCM band.
bands : `list` [`str`]
List of band names from FGCM output
Returns
-------
offsets : `numpy.array` of floats
Per band zeropoint offsets
"""
# Only use stars that are observed in all the bands that were actually used
# This will ensure that we use the same healpix pixels for the absolute
# calibration of each band.
minObs = stdCat['ngood'].min(axis=1)
goodStars = (minObs >= 1)
stdCat = stdCat[goodStars]
self.log.info(
"Found %d stars with at least 1 good observation in each band" %
(len(stdCat)))
# Associate each band with the appropriate physicalFilter and make
# filterLabels
filterLabels = []
lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',')
lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',')
physicalFilterMapBands = list(physicalFilterMap.values())
physicalFilterMapFilters = list(physicalFilterMap.keys())
for band in bands:
# Find a physical filter associated from the band by doing
# a reverse lookup on the physicalFilterMap dict
physicalFilterMapIndex = physicalFilterMapBands.index(band)
physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex]
# Find the appropriate fgcm standard physicalFilter
lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter)
stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex]
filterLabels.append(
afwImage.FilterLabel(band=band, physical=stdPhysicalFilter))
# We have to make a table for each pixel with flux/fluxErr
# This is a temporary table generated for input to the photoCal task.
# These fluxes are not instFlux (they are top-of-the-atmosphere approximate and
# have had chromatic corrections applied to get to the standard system
# specified by the atmosphere/instrumental parameters), nor are they
# in Jansky (since they don't have a proper absolute calibration: the overall
# zeropoint is estimated from the telescope size, etc.)
sourceMapper = afwTable.SchemaMapper(stdCat.schema)
sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
sourceMapper.editOutputSchema().addField(
'instFlux', type=np.float64, doc="instrumental flux (counts)")
sourceMapper.editOutputSchema().addField(
'instFluxErr',
type=np.float64,
doc="instrumental flux error (counts)")
badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar',
type='Flag',
doc="bad flag")
# Split up the stars
# Note that there is an assumption here that the ra/dec coords stored
# on-disk are in radians, and therefore that starObs['coord_ra'] /
# starObs['coord_dec'] return radians when used as an array of numpy float64s.
theta = np.pi / 2. - stdCat['coord_dec']
phi = stdCat['coord_ra']
ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi)
h, rev = esutil.stat.histogram(ipring, rev=True)
gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
self.log.info(
"Found %d pixels (nside=%d) with at least %d good stars" %
(gdpix.size, self.config.referencePixelizationNside,
self.config.referencePixelizationMinStars))
if gdpix.size < self.config.referencePixelizationNPixels:
self.log.warning(
"Found fewer good pixels (%d) than preferred in configuration (%d)"
% (gdpix.size, self.config.referencePixelizationNPixels))
else:
# Sample out the pixels we want to use
gdpix = np.random.choice(
gdpix,
size=self.config.referencePixelizationNPixels,
replace=False)
results = np.zeros(gdpix.size,
dtype=[('hpix', 'i4'), ('nstar', 'i4', len(bands)),
('nmatch', 'i4', len(bands)),
('zp', 'f4', len(bands)),
('zpErr', 'f4', len(bands))])
results['hpix'] = ipring[rev[rev[gdpix]]]
# We need a boolean index to deal with catalogs...
selected = np.zeros(len(stdCat), dtype=bool)
refFluxFields = [None] * len(bands)
for p_index, pix in enumerate(gdpix):
i1a = rev[rev[pix]:rev[pix + 1]]
# the stdCat afwTable can only be indexed with boolean arrays,
# and not numpy index arrays (see DM-16497). This little trick
# converts the index array into a boolean array
selected[:] = False
selected[i1a] = True
for b_index, filterLabel in enumerate(filterLabels):
struct = self._computeOffsetOneBand(sourceMapper, badStarKey,
b_index, filterLabel,
stdCat, selected,
refFluxFields)
results['nstar'][p_index, b_index] = len(i1a)
results['nmatch'][p_index, b_index] = len(struct.arrays.refMag)
results['zp'][p_index, b_index] = struct.zp
results['zpErr'][p_index, b_index] = struct.sigma
# And compute the summary statistics
offsets = np.zeros(len(bands))
for b_index, band in enumerate(bands):
# make configurable
ok, = np.where(
results['nmatch'][:, b_index] >= self.config.referenceMinMatch)
offsets[b_index] = np.median(results['zp'][ok, b_index])
# use median absolute deviation to estimate Normal sigma
# see https://en.wikipedia.org/wiki/Median_absolute_deviation
madSigma = 1.4826 * np.median(
np.abs(results['zp'][ok, b_index] - offsets[b_index]))
self.log.info(
"Reference catalog offset for %s band: %.12f +/- %.12f", band,
offsets[b_index], madSigma)
return offsets
def _computeOffsetOneBand(self, sourceMapper, badStarKey, b_index,
filterLabel, stdCat, selected, refFluxFields):
"""
Compute the zeropoint offset between the fgcm stdCat and the reference
stars for one pixel in one band
Parameters
----------
sourceMapper : `lsst.afw.table.SchemaMapper`
Mapper to go from stdCat to calibratable catalog
badStarKey : `lsst.afw.table.Key`
Key for the field with bad stars
b_index : `int`
Index of the band in the star catalog
filterLabel : `lsst.afw.image.FilterLabel`
filterLabel with band and physical filter
stdCat : `lsst.afw.table.SimpleCatalog`
FGCM standard stars
selected : `numpy.array(dtype=bool)`
Boolean array of which stars are in the pixel
refFluxFields : `list`
List of names of flux fields for reference catalog
"""
sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
sourceCat.reserve(selected.sum())
sourceCat.extend(stdCat[selected], mapper=sourceMapper)
sourceCat['instFlux'] = 10.**(
stdCat['mag_std_noabs'][selected, b_index] / (-2.5))
sourceCat['instFluxErr'] = (np.log(10.) / 2.5) * (
stdCat['magErr_std'][selected, b_index] * sourceCat['instFlux'])
# Make sure we only use stars that have valid measurements
# (This is perhaps redundant with requirements above that the
# stars be observed in all bands, but it can't hurt)
badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0)
for rec in sourceCat[badStar]:
rec.set(badStarKey, True)
exposure = afwImage.ExposureF()
exposure.setFilterLabel(filterLabel)
if refFluxFields[b_index] is None:
# Need to find the flux field in the reference catalog
# to work around limitations of DirectMatch in PhotoCal
ctr = stdCat[0].getCoord()
rad = 0.05 * lsst.geom.degrees
refDataTest = self.refObjLoader.loadSkyCircle(
ctr, rad, filterLabel.bandLabel)
refFluxFields[b_index] = refDataTest.fluxField
# Make a copy of the config so that we can modify it
calConfig = copy.copy(self.config.photoCal.value)
calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[
b_index]
calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[
b_index] + 'Err'
calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader,
config=calConfig,
schema=sourceCat.getSchema())
struct = calTask.run(exposure, sourceCat)
return struct
def _formatCatalog(self, fgcmStarCat, offsets, bands):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets : `list` with len(self.bands) entries
Zeropoint offsets to apply
bands : `list` [`str`]
List of band names from FGCM output
Returns
-------
formattedCat: `lsst.afw.table.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nTotal' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' %
(band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(bands):
mag = fgcmStarCat['mag_std_noabs'][:, b].astype(
np.float64) + offsets[b]
# We want fluxes in nJy from calibrated AB magnitudes
# (after applying offset). Updated after RFC-549 and RFC-575.
flux = (mag * units.ABmag).to_value(units.nJy)
fluxErr = (np.log(10.) /
2.5) * flux * fgcmStarCat['magErr_std'][:, b].astype(
np.float64)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
formattedCat['%s_nPsfCandidate' %
(band)][:] = fgcmStarCat['npsfcand'][:, b]
addRefCatMetadata(formattedCat)
return formattedCat
def _outputZeropoints(self,
camera,
zptCat,
visitCat,
offsets,
bands,
physicalFilterMap,
tract=None):
"""Output the zeropoints in fgcm_photoCalib format.
Parameters
----------
camera : `lsst.afw.cameraGeom.Camera`
Camera from the butler.
zptCat : `lsst.afw.table.BaseCatalog`
FGCM zeropoint catalog from `FgcmFitCycleTask`.
visitCat : `lsst.afw.table.BaseCatalog`
FGCM visitCat from `FgcmBuildStarsTask`.
offsets : `numpy.array`
Float array of absolute calibration offsets, one for each filter.
bands : `list` [`str`]
List of band names from FGCM output.
physicalFilterMap : `dict`
Dictionary of mappings from physical filter to FGCM band.
tract: `int`, optional
Tract number to output. Default is None (global calibration)
Returns
-------
photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)]
Generator that returns (visit, exposureCatalog) tuples.
"""
# Select visit/ccds where we have a calibration
# This includes ccds where we were able to interpolate from neighboring
# ccds.
cannot_compute = fgcm.fgcmUtilities.zpFlagDict[
'CANNOT_COMPUTE_ZEROPOINT']
selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0)
& (zptCat['fgcmZptVar'] > 0.0)
& (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE))
# Log warnings for any visit which has no valid zeropoints
badVisits = np.unique(zptCat['visit'][~selected])
goodVisits = np.unique(zptCat['visit'][selected])
allBadVisits = badVisits[~np.isin(badVisits, goodVisits)]
for allBadVisit in allBadVisits:
self.log.warning(f'No suitable photoCalib for visit {allBadVisit}')
# Get a mapping from filtername to the offsets
offsetMapping = {}
for f in physicalFilterMap:
# Not every filter in the map will necesarily have a band.
if physicalFilterMap[f] in bands:
offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])]
# Get a mapping from "ccd" to the ccd index used for the scaling
ccdMapping = {}
for ccdIndex, detector in enumerate(camera):
ccdMapping[detector.getId()] = ccdIndex
# And a mapping to get the flat-field scaling values
scalingMapping = {}
for rec in visitCat:
scalingMapping[rec['visit']] = rec['scaling']
if self.config.doComposeWcsJacobian:
approxPixelAreaFields = computeApproxPixelAreaFields(camera)
# The zptCat is sorted by visit, which is useful
lastVisit = -1
zptVisitCatalog = None
metadata = dafBase.PropertyList()
metadata.add("COMMENT", "Catalog id is detector id, sorted.")
metadata.add("COMMENT", "Only detectors with data have entries.")
for rec in zptCat[selected]:
# Retrieve overall scaling
scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]]
# The postCalibrationOffset describe any zeropoint offsets
# to apply after the fgcm calibration. The first part comes
# from the reference catalog match (used in testing). The
# second part comes from the mean chromatic correction
# (if configured).
postCalibrationOffset = offsetMapping[rec['filtername']]
if self.config.doApplyMeanChromaticCorrection:
postCalibrationOffset += rec['fgcmDeltaChrom']
fgcmSuperStarField = self._getChebyshevBoundedField(
rec['fgcmfZptSstarCheb'], rec['fgcmfZptChebXyMax'])
# Convert from FGCM AB to nJy
fgcmZptField = self._getChebyshevBoundedField(
(rec['fgcmfZptCheb'] * units.AB).to_value(units.nJy),
rec['fgcmfZptChebXyMax'],
offset=postCalibrationOffset,
scaling=scaling)
if self.config.doComposeWcsJacobian:
fgcmField = afwMath.ProductBoundedField([
approxPixelAreaFields[rec['detector']], fgcmSuperStarField,
fgcmZptField
])
else:
# The photoCalib is just the product of the fgcmSuperStarField and the
# fgcmZptField
fgcmField = afwMath.ProductBoundedField(
[fgcmSuperStarField, fgcmZptField])
# The "mean" calibration will be set to the center of the ccd for reference
calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter())
calibErr = (np.log(10.0) / 2.5) * calibCenter * np.sqrt(
rec['fgcmZptVar'])
photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter,
calibrationErr=calibErr,
calibration=fgcmField,
isConstant=False)
# Return full per-visit exposure catalogs
if rec['visit'] != lastVisit:
# This is a new visit. If the last visit was not -1, yield
# the ExposureCatalog
if lastVisit > -1:
# ensure that the detectors are in sorted order, for fast lookups
zptVisitCatalog.sort()
yield (int(lastVisit), zptVisitCatalog)
else:
# We need to create a new schema
zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema(
)
zptExpCatSchema.addField('visit',
type='L',
doc='Visit number')
# And start a new one
zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema)
zptVisitCatalog.setMetadata(metadata)
lastVisit = int(rec['visit'])
catRecord = zptVisitCatalog.addNew()
catRecord['id'] = int(rec['detector'])
catRecord['visit'] = rec['visit']
catRecord.setPhotoCalib(photoCalib)
# Final output of last exposure catalog
# ensure that the detectors are in sorted order, for fast lookups
zptVisitCatalog.sort()
yield (int(lastVisit), zptVisitCatalog)
def _getChebyshevBoundedField(self,
coefficients,
xyMax,
offset=0.0,
scaling=1.0):
"""
Make a ChebyshevBoundedField from fgcm coefficients, with optional offset
and scaling.
Parameters
----------
coefficients: `numpy.array`
Flattened array of chebyshev coefficients
xyMax: `list` of length 2
Maximum x and y of the chebyshev bounding box
offset: `float`, optional
Absolute calibration offset. Default is 0.0
scaling: `float`, optional
Flat scaling value from fgcmBuildStars. Default is 1.0
Returns
-------
boundedField: `lsst.afw.math.ChebyshevBoundedField`
"""
orderPlus1 = int(np.sqrt(coefficients.size))
pars = np.zeros((orderPlus1, orderPlus1))
bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0),
lsst.geom.Point2I(*xyMax))
pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) *
(10.**(offset / -2.5)) * scaling)
boundedField = afwMath.ChebyshevBoundedField(bbox, pars)
return boundedField
def _outputAtmospheres(self, handleDict, atmCat):
"""
Output the atmospheres.
Parameters
----------
handleDict : `dict`
All data handles are `lsst.daf.butler.DeferredDatasetHandle`
The handleDict has the follownig keys:
``"fgcmLookUpTable"``
handle for the FGCM look-up table.
atmCat : `lsst.afw.table.BaseCatalog`
FGCM atmosphere parameter catalog from fgcmFitCycleTask.
Returns
-------
atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)]
Generator that returns (visit, transmissionCurve) tuples.
"""
# First, we need to grab the look-up table and key info
lutCat = handleDict['fgcmLookUpTable'].get()
atmosphereTableName = lutCat[0]['tablename']
elevation = lutCat[0]['elevation']
atmLambda = lutCat[0]['atmLambda']
lutCat = None
# Make the atmosphere table if possible
try:
atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(
atmosphereTableName)
atmTable.loadTable()
except IOError:
atmTable = None
if atmTable is None:
# Try to use MODTRAN instead
try:
modGen = fgcm.ModtranGenerator(elevation)
lambdaRange = np.array([atmLambda[0], atmLambda[-1]]) / 10.
lambdaStep = (atmLambda[1] - atmLambda[0]) / 10.
except (ValueError, IOError) as e:
raise RuntimeError(
"FGCM look-up-table generated with modtran, "
"but modtran not configured to run.") from e
zenith = np.degrees(np.arccos(1. / atmCat['secZenith']))
for i, visit in enumerate(atmCat['visit']):
if atmTable is not None:
# Interpolate the atmosphere table
atmVals = atmTable.interpolateAtmosphere(
pmb=atmCat[i]['pmb'],
pwv=atmCat[i]['pwv'],
o3=atmCat[i]['o3'],
tau=atmCat[i]['tau'],
alpha=atmCat[i]['alpha'],
zenith=zenith[i],
ctranslamstd=[atmCat[i]['cTrans'], atmCat[i]['lamStd']])
else:
# Run modtran
modAtm = modGen(
pmb=atmCat[i]['pmb'],
pwv=atmCat[i]['pwv'],
o3=atmCat[i]['o3'],
tau=atmCat[i]['tau'],
alpha=atmCat[i]['alpha'],
zenith=zenith[i],
lambdaRange=lambdaRange,
lambdaStep=lambdaStep,
ctranslamstd=[atmCat[i]['cTrans'], atmCat[i]['lamStd']])
atmVals = modAtm['COMBINED']
# Now need to create something to persist...
curve = TransmissionCurve.makeSpatiallyConstant(
throughput=atmVals,
wavelengths=atmLambda,
throughputAtMin=atmVals[0],
throughputAtMax=atmVals[-1])
yield (int(visit), curve)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = butlerQC.get(inputRefs)
self.log.info("Running with %d sourceTable_visit handles",
(len(handleDict['source_catalogs'])))
# Run the build stars tasks
tract = butlerQC.quantum.dataId['tract']
handleDict['sourceSchema'] = self.sourceSchema
sourceTableHandles = handleDict['source_catalogs']
sourceTableHandleDict = {
sourceTableHandle.dataId['visit']: sourceTableHandle
for sourceTableHandle in sourceTableHandles
}
visitSummaryHandles = handleDict['visitSummary']
visitSummaryHandleDict = {
visitSummaryHandle.dataId['visit']: visitSummaryHandle
for visitSummaryHandle in visitSummaryHandles
}
handleDict['sourceTableHandleDict'] = sourceTableHandleDict
handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict
# And the outputs
if self.config.fgcmOutputProducts.doZeropointOutput:
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
handleDict['fgcmPhotoCalibs'] = photoCalibRefDict
if self.config.fgcmOutputProducts.doAtmosphereOutput:
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
handleDict['fgcmTransmissionAtmospheres'] = atmRefDict
if self.config.fgcmBuildStars.doReferenceMatches:
refConfig = LoadReferenceObjectsConfig()
refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap
loader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
config=refConfig,
log=self.log)
buildStarsRefObjLoader = loader
else:
buildStarsRefObjLoader = None
if self.config.fgcmOutputProducts.doReferenceCalibration:
refConfig = self.config.fgcmOutputProducts.refObjLoader
loader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
config=refConfig,
log=self.log)
self.fgcmOutputProducts.refObjLoader = loader
struct = self.run(handleDict,
tract,
buildStarsRefObjLoader=buildStarsRefObjLoader)
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
# Turn raw repeatability into simple catalog for persistence
schema = afwTable.Schema()
schema.addField('rawRepeatability',
type=np.float64,
doc="Per-band raw repeatability in FGCM calibration.")
repeatabilityCat = afwTable.BaseCatalog(schema)
repeatabilityCat.resize(len(struct.repeatability))
repeatabilityCat['rawRepeatability'][:] = struct.repeatability
butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
return
def test_fgcmLoadReferenceOtherFilters(self):
"""
Test loading of the fgcm reference catalogs using unmatched filter names.
"""
filterList = ['HSC-R2', 'HSC-I2']
config = fgcmcal.FgcmLoadReferenceCatalogConfig()
config.applyColorTerms = True
config.filterMap = {'HSC-R2': 'r', 'HSC-I2': 'i'}
config.colorterms.data = {}
config.colorterms.data[
'ps1*'] = lsst.pipe.tasks.colorterms.ColortermDict()
config.colorterms.data['ps1*'].data = {}
config.colorterms.data['ps1*'].data[
'HSC-R2'] = lsst.pipe.tasks.colorterms.Colorterm()
config.colorterms.data['ps1*'].data['HSC-R2'].primary = 'r'
config.colorterms.data['ps1*'].data['HSC-R2'].secondary = 'i'
config.colorterms.data['ps1*'].data['HSC-R2'].c0 = -0.000032
config.colorterms.data['ps1*'].data['HSC-R2'].c1 = -0.002866
config.colorterms.data['ps1*'].data['HSC-R2'].c2 = -0.012638
config.colorterms.data['ps1*'].data[
'HSC-I2'] = lsst.pipe.tasks.colorterms.Colorterm()
config.colorterms.data['ps1*'].data['HSC-I2'].primary = 'i'
config.colorterms.data['ps1*'].data['HSC-I2'].secondary = 'z'
config.colorterms.data['ps1*'].data['HSC-I2'].c0 = 0.001625
config.colorterms.data['ps1*'].data['HSC-I2'].c1 = -0.200406
config.colorterms.data['ps1*'].data['HSC-I2'].c2 = -0.013666
refCatName = 'ps1_pv3_3pi_20170110'
butler = lsst.daf.butler.Butler(
self.repo,
instrument='HSC',
collections=['HSC/testdata', 'refcats/gen2'])
refs = set(butler.registry.queryDatasets(refCatName))
dataIds = [butler.registry.expandDataId(ref.dataId) for ref in refs]
refCats = [butler.getDirectDeferred(ref) for ref in refs]
refConfig = LoadReferenceObjectsConfig()
refConfig.filterMap = config.filterMap
refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig)
loadCat = fgcmcal.FgcmLoadReferenceCatalogTask(
refObjLoader=refObjLoader, refCatName=refCatName, config=config)
ra = 337.656174
dec = 0.823595
rad = 0.1
refCat = loadCat.getFgcmReferenceStarsSkyCircle(
ra, dec, rad, filterList)
self.assertEqual(len(filterList), refCat['refMag'].shape[1])
self.assertEqual(len(filterList), refCat['refMagErr'].shape[1])
test, = np.where((refCat['refMag'][:, 0] < 30.0)
& (refCat['refMag'][:, 1] < 30.0))
self.assertGreater(test.size, 0)
class GenerateDonutCatalogOnlineTask(pipeBase.Task):
"""
Construct a source catalog from reference catalogs
and pointing information.
Parameters
----------
dataIds : `list`
List of the dataIds for the reference catalog shards.
refCats : `list`
List of the deferred dataset references pointing to the pieces
of the reference catalog we want in the butler.
**kwargs : dict[str, any]
Dictionary of input argument: new value for that input argument.
"""
ConfigClass = GenerateDonutCatalogOnlineTaskConfig
_DefaultName = "generateDonutCatalogOnlineTask"
def __init__(self, dataIds, refCats, **kwargs):
super().__init__(**kwargs)
refConfig = self.config.refObjLoader
# refObjLoader handles the interaction with the butler repository
# needed to get the pieces of the reference catalogs we need.
self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
refCats=refCats,
config=refConfig,
log=self.log)
if self.config.doReferenceSelection:
self.makeSubtask("referenceSelector")
self.filterName = self.config.filterName
self.config.refObjLoader.pixelMargin = 0
self.config.refObjLoader.anyFilterMapsToThis = self.filterName
self.config.referenceSelector.magLimit.fluxField = f"{self.filterName}_flux"
self.config.referenceSelector.signalToNoise.fluxField = (
f"{self.filterName}_flux")
self.config.donutSelector.fluxField = f"{self.filterName}_flux"
if self.config.doDonutSelection:
self.makeSubtask("donutSelector")
@timeMethod
def run(self, detector, wcs):
"""Get the data from the reference catalog only from the
shards of the reference catalogs that overlap our pointing.
Parameters
----------
detector : `lsst.afw.cameraGeom.Detector`
Detector object from the camera.
wcs : `lsst.afw.geom.SkyWcs`
Wcs object defining the pixel to sky (and inverse) transform for
the supplied ``bbox``.
Returns
-------
struct : `lsst.pipe.base.Struct`
The struct contains the following data:
- DonutCatalog: `pandas.DataFrame`
The final donut source catalog for the region.
"""
bbox = detector.getBBox()
# Get the refcatalog shard
skyBox = self.refObjLoader.loadPixelBox(bbox,
wcs,
filterName=self.filterName,
bboxToSpherePadding=0)
if not skyBox.refCat.isContiguous():
refCat = skyBox.refCat.copy(deep=True)
else:
refCat = skyBox.refCat
donutCatalog = self._formatCatalog(refCat, detector)
return pipeBase.Struct(donutCatalog=donutCatalog)
def _formatCatalog(self, refCat, detector):
"""Format a reference afw table into the final format.
Parameters
----------
refCat : `lsst.afw.table.SourceCatalog`
Reference catalog in afw format.
detector : `lsst.afw.cameraGeom.Detector`
Detector object from the camera.
Returns
-------
refCat : `pandas.DataFrame`
Reference catalog.
"""
if self.config.doReferenceSelection:
goodSources = self.referenceSelector.selectSources(refCat)
refSelected = goodSources.selected
else:
refSelected = np.ones(len(refCat), dtype=bool)
if self.config.doDonutSelection:
self.log.info("Running Donut Selector")
donutSelection = self.donutSelector.run(refCat, detector)
donutSelected = donutSelection.selected
else:
donutSelected = np.ones(len(refCat), dtype=bool)
selected = refSelected & donutSelected
npRefCat = np.zeros(
np.sum(selected),
dtype=[
("coord_ra", "f8"),
("coord_dec", "f8"),
("centroid_x", "f8"),
("centroid_y", "f8"),
("source_flux", "f8"),
],
)
if npRefCat.size == 0:
# Return an empty catalog if we don't have any selected sources.
return npRefCat
# Natively "coord_ra" and "coord_dec" are stored in radians.
# Doing this as an array rather than by row with the coord access is
# approximately 600x faster.
npRefCat["coord_ra"] = refCat["coord_ra"][selected]
npRefCat["coord_dec"] = refCat["coord_dec"][selected]
npRefCat["centroid_x"] = refCat["centroid_x"][selected]
npRefCat["centroid_y"] = refCat["centroid_y"][selected]
fluxField = f"{self.filterName}_flux"
# nan_to_num replaces nans with zeros, and this ensures that
# we select fluxes that both filter out nans and are positive.
(good, ) = np.where(
(np.nan_to_num(refCat[fluxField][selected]) > 0.0)
& (np.nan_to_num(refCat[fluxField + "Err"][selected]) > 0.0))
npRefCat["source_flux"][good] = refCat[fluxField][selected][good]
return pd.DataFrame.from_records(npRefCat)
def runTest(withRaDecErr):
# Generate a second catalog, with different ids
inTempDir1 = tempfile.TemporaryDirectory()
inPath1 = inTempDir1.name
skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1,
idStart=25,
seed=123)
inTempDir2 = tempfile.TemporaryDirectory()
inPath2 = inTempDir2.name
skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2,
idStart=5432,
seed=11)
# override some field names, and use multiple cores
config = ingestIndexTestBase.makeConvertConfig(
withRaDecErr=withRaDecErr,
withMagErr=True,
withPm=True,
withPmErr=True)
# use a very small HTM pixelization depth to ensure there will be collisions when
# ingesting the files in parallel
depth = 2
config.dataset_config.indexer.active.depth = depth
# np.savetxt prepends '# ' to the header lines, so use a reader that understands that
config.file_reader.format = 'ascii.commented_header'
config.n_processes = 2 # use multiple cores for this test only
config.id_name = 'id' # Use the ids from the generated catalogs
repoPath = os.path.join(
self.outPath, "output_multifile_parallel",
"_withRaDecErr" if withRaDecErr else "_noRaDecErr")
# Convert the input data files to our HTM indexed format.
dataTempDir = tempfile.TemporaryDirectory()
dataPath = dataTempDir.name
converter = ConvertReferenceCatalogTask(output_dir=dataPath,
config=config)
converter.run([skyCatalogFile1, skyCatalogFile2])
# Make a temporary butler to ingest them into.
butler = self.makeTemporaryRepo(
repoPath, config.dataset_config.indexer.active.depth)
dimensions = [f"htm{depth}"]
datasetType = DatasetType(config.dataset_config.ref_dataset_name,
dimensions,
"SimpleCatalog",
universe=butler.registry.dimensions,
isCalibration=False)
butler.registry.registerDatasetType(datasetType)
# Ingest the files into the new butler.
run = "testingRun"
htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
ingest_files(repoPath,
config.dataset_config.ref_dataset_name,
run,
htmTableFile,
transfer="auto")
# Test if we can get back the catalogs, with a new butler.
butler = lsst.daf.butler.Butler(repoPath)
datasetRefs = list(
butler.registry.queryDatasets(
config.dataset_config.ref_dataset_name,
collections=[run]).expanded())
handlers = []
for dataRef in datasetRefs:
handlers.append(
DeferredDatasetHandle(butler=butler,
ref=dataRef,
parameters=None))
loaderConfig = ReferenceObjectLoader.ConfigClass()
loader = ReferenceObjectLoader(
[dataRef.dataId for dataRef in datasetRefs],
handlers,
config=loaderConfig,
log=self.logger)
self.checkAllRowsInRefcat(loader, skyCatalog1, config)
self.checkAllRowsInRefcat(loader, skyCatalog2, config)
inTempDir1.cleanup()
inTempDir2.cleanup()
dataTempDir.cleanup()