def setUp(self):
refCatDir = os.path.join(os.path.dirname(__file__), "data",
"sdssrefcat")
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(3001, 3001))
self.ctrPix = afwGeom.Point2I(1500, 1500)
metadata = dafBase.PropertySet()
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CUNIT1", "deg")
metadata.set("CUNIT2", "deg")
metadata.set("CRVAL1", 215.5)
metadata.set("CRVAL2", 53.0)
metadata.set("CRPIX1", self.ctrPix[0] + 1)
metadata.set("CRPIX2", self.ctrPix[1] + 1)
metadata.set("CD1_1", 5.1e-05)
metadata.set("CD1_2", 0.0)
metadata.set("CD2_2", -5.1e-05)
metadata.set("CD2_1", 0.0)
self.tanWcs = afwImage.makeWcs(metadata)
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(self.tanWcs)
self.exposure.setFilter(afwImage.Filter("r", True))
butler = Butler(refCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def testLoadPixelBox(self):
"""Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config."""
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
numFound = 0
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary
ctr_pix = bbox.getCenter()
# catalog is sparse, so set pixel scale such that bbox encloses region
# used to generate compCats
pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth())
cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale)
wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix)
result = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a")
# The following is to ensure the reference catalog coords are
# getting corrected for proper motion when an epoch is provided.
# Use an extreme epoch so that differences in corrected coords
# will be significant. Note that this simply tests that the coords
# do indeed change when the epoch is passed. It makes no attempt
# at assessing the correctness of the change. This is left to the
# explicit testProperMotion() test below.
resultWithEpoch = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a",
epoch=astropy.time.Time(20000, format='mjd', scale="tai"))
self.assertFloatsNotEqual(result.refCat["coord_ra"], resultWithEpoch.refCat["coord_ra"],
rtol=1.0e-4)
self.assertFloatsNotEqual(result.refCat["coord_dec"], resultWithEpoch.refCat["coord_dec"],
rtol=1.0e-4)
self.assertFalse("camFlux" in result.refCat.schema)
self.assertGreaterEqual(len(result.refCat), len(idList))
numFound += len(result.refCat)
self.assertGreater(numFound, 0)
def validateMatches(self, dataId):
sources = self.butler.get(self._sourceDataset, dataId)
packedMatches = self.butler.get(self._matchDataset, dataId)
config = LoadIndexedReferenceObjectsTask.ConfigClass()
config.ref_dataset_name = "ps1_pv3_3pi_20170110"
refObjLoader = LoadIndexedReferenceObjectsTask(self.butler, config=config)
matches = refObjLoader.joinMatchListWithCatalog(packedMatches, sources)
self.assertGreater("Number of matches", len(matches), self._minMatches)
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", self.outPath+"/output_multifile",
self.skyCatalogFile, self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(self.outPath+"/output_multifile"))
self.assertEqual(loader.dataset_config.format_version, 1)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
def setUp(self):
np.random.seed(12345)
self.butler = Butler(RefCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=self.butler)
center = lsst.afw.geom.SpherePoint(215.5, 53.0, lsst.afw.geom.degrees)
radius = 0.5 * lsst.afw.geom.degrees
self.filter = "r"
self.references = refObjLoader.loadSkyCircle(center, radius,
self.filter).refCat
def ref_task(self):
"""
Handle for the ReferenceObjectsTask.
"""
if self._ref_task is None:
refConfig = LoadIndexedReferenceObjectsTask.ConfigClass()
refConfig.filterMap = {_: f'lsst_{_}_smeared' for _ in 'ugrizy'}
self._ref_task = LoadIndexedReferenceObjectsTask(self.butler,
config=refConfig)
return self._ref_task
def testLoadVersion1(self):
"""Test reading a format_version=1 catalog (fluxes unchanged)."""
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version1')
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
self.assertEqual(loader.dataset_config.format_version, 1)
result = loader.loadSkyCircle(make_coord(10, 20), 5*lsst.geom.degrees, filterName='a')
self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
def __init__(self, butler=None, **kwargs):
"""!
@param[in] butler The butler is passed to the refObjLoader constructor in case it is
needed. Ignored if the refObjLoader argument provides a loader directly.
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
pipeBase.CmdLineTask.__init__(self, **kwargs)
self.butler = butler
# Configure LoadIndexedReferenceObjectsTask
refConfig = LoadIndexedReferenceObjectsTask.ConfigClass()
refConfig.ref_dataset_name = self.config.refCat
self.refTask = LoadIndexedReferenceObjectsTask(self.butler, config=refConfig)
def setUp(self):
refCatDir = os.path.join(os.path.dirname(__file__), "data", "sdssrefcat")
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(3001, 3001))
crpix = afwGeom.Box2D(self.bbox).getCenter()
self.tanWcs = afwGeom.makeSkyWcs(crpix=crpix,
crval=afwGeom.SpherePoint(215.5, 53.0, afwGeom.degrees),
cdMatrix=afwGeom.makeCdMatrix(scale=5.1e-5*afwGeom.degrees))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(self.tanWcs)
self.exposure.setFilter(afwImage.Filter("r", True))
butler = Butler(refCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def testLoadVersion1(self):
"""Test reading a format_version=1 catalog (fluxes unchanged)."""
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version1')
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
self.assertEqual(loader.dataset_config.format_version, 1)
result = loader.loadSkyCircle(make_coord(10, 20),
5*lsst.geom.degrees, filterName='a')
self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
def testLoadVersion0(self):
"""Test reading a pre-written format_version=0 (Jy flux) catalog.
It should be converted to have nJy fluxes.
"""
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version0')
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
self.assertEqual(loader.dataset_config.format_version, 0)
result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20),
5*lsst.geom.degrees, filterName='a')
self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux'])
self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr'])
self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux'])
self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr'])
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits"))
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate
# create an exposure with the right metadata; the closest thing we have is
# apparently v695833-e0-c000-a00.sci.fits, which is much too small
smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits"))
self.exposure = ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
# copy the pixels we can, in case the user wants a debug display
mi = self.exposure.getMaskedImage()
mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox())
logLevel = Log.INFO
refCatDir = os.path.join(testDir, "data", "sdssrefcat")
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
astrometryConfig = AstrometryTask.ConfigClass()
self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader)
self.astrom.log.setLevel(logLevel)
# Since our sourceSelector is a registry object we have to wait for it to be created
# before setting default values.
self.astrom.matcher.sourceSelector.config.minSnr = 0
def runTest(withRaDecErr):
# Generate a second catalog, with different ids
inPath1 = tempfile.mkdtemp()
skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
inPath2 = tempfile.mkdtemp()
skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
# override some field names, and use multiple cores
config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# use a very small HTM pixelization depth to ensure there will be collisions when
# ingesting the files in parallel
config.dataset_config.indexer.active.depth = 2
# np.savetxt prepends '# ' to the header lines, so use a reader that understands that
config.file_reader.format = 'ascii.commented_header'
config.n_processes = 2 # use multiple cores for this test only
config.id_name = 'id' # Use the ids from the generated catalogs
outpath = os.path.join(self.outPath, "output_multifile_parallel",
"_withRaDecErr" if withRaDecErr else "")
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outpath,
skyCatalogFile1, skyCatalogFile2], config=config)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(outpath)
loaderConfig = LoadIndexedReferenceObjectsConfig()
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, skyCatalog1, config)
self.checkAllRowsInRefcat(loader, skyCatalog2, config)
def testDefaultFilterAndFilterMap(self):
"""Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
config = LoadIndexedReferenceObjectsConfig()
config.defaultFilter = "b"
config.filterMap = {"aprime": "a"}
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius)
self.assertEqual(lcat.fluxField, "camFlux")
if len(idList) > 0:
defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
self.assertTrue(defFluxFieldName in lcat.refCat.schema)
aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
break # just need one test
def testDefaultFilterAndFilterMap(self):
"""Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
config = LoadIndexedReferenceObjectsConfig()
config.defaultFilter = "b"
config.filterMap = {"aprime": "a"}
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius)
self.assertEqual(lcat.fluxField, "camFlux")
if len(idList) > 0:
defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
self.assertTrue(defFluxFieldName in lcat.refCat.schema)
aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
break # just need one test
class RefCat:
def __init__(self, butler):
self.butler = butler
refConfig = LoadIndexedReferenceObjectsTask.ConfigClass()
self.refTask = LoadIndexedReferenceObjectsTask(self.butler,
config=refConfig)
def get_pixel_coords(self, dataId, mag_cut=22.):
calexp = self.butler.get('calexp', dataId)
wcs = calexp.getWcs()
dim = calexp.getDimensions()
centerPixel = afw_geom.Point2D(dim.getX()/2., dim.getY()/2.)
centerCoord = wcs.pixelToSky(centerPixel)
radius = afw_geom.Angle(0.17, afw_geom.degrees)
ref_cat \
= self.refTask.loadSkyCircle(centerCoord, radius,
calexp.getFilter().getName()).refCat
xref, yref = [], []
mags = -2.5*np.log10(ref_cat['u_flux']/3631.)
for i, row in enumerate(ref_cat):
if mags[i] > mag_cut:
continue
point = wcs.skyToPixel(row.getCoord())
xref.append(point.getX())
yref.append(point.getY())
return xref, yref, ref_cat
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcCat = afwTable.SourceCatalog.readFits(
os.path.join(testDir, "data", "v695833-e0-c000.xy.fits"))
self.srcCat["slot_ApFlux_fluxSigma"] = 1
self.srcCat["slot_PsfFlux_fluxSigma"] = 1
# The .xy.fits file has sources in the range ~ [0,2000],[0,4500]
# which is bigger than the exposure
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(2048, 4612))
smallExposure = afwImage.ExposureF(
os.path.join(testDir, "data", "v695833-e0-c000-a00.sci.fits"))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
self.exposure.setCalib(smallExposure.getCalib())
# Make a reference loader
butler = Butler(RefCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
logLevel = Log.TRACE
self.log = Log.getLogger('testPhotoCal')
self.log.setLevel(logLevel)
self.config = PhotoCalConfig()
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
self.config.fluxField = "base_PsfFlux_flux"
def _formatCatalog(self, fgcmStarCat, offsets, bands):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat : `lsst.afw.Table.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets : `list` with len(self.bands) entries
Zeropoint offsets to apply
bands : `list` [`str`]
List of band names from FGCM output
Returns
-------
formattedCat: `lsst.afw.table.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nTotal' % (band),
type=np.int32)
sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' %
(band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(bands):
mag = fgcmStarCat['mag_std_noabs'][:, b].astype(
np.float64) + offsets[b]
# We want fluxes in nJy from calibrated AB magnitudes
# (after applying offset). Updated after RFC-549 and RFC-575.
flux = (mag * units.ABmag).to_value(units.nJy)
fluxErr = (np.log(10.) /
2.5) * flux * fgcmStarCat['magErr_std'][:, b].astype(
np.float64)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b]
formattedCat['%s_nPsfCandidate' %
(band)][:] = fgcmStarCat['npsfcand'][:, b]
addRefCatMetadata(formattedCat)
return formattedCat
def testProperMotion(self):
"""Test proper motion correction"""
center = make_coord(93.0, -90.0)
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
references = loader.loadSkyCircle(center, self.searchRadius, filterName='a').refCat
original = references.copy(True)
# Zero epoch change --> no proper motion correction (except minor numerical effects)
loader.applyProperMotions(references, self.epoch)
self.assertFloatsAlmostEqual(references["coord_ra"], original["coord_ra"], rtol=1.0e-14)
self.assertFloatsAlmostEqual(references["coord_dec"], original["coord_dec"], rtol=1.0e-14)
self.assertFloatsEqual(references["coord_raErr"], original["coord_raErr"])
self.assertFloatsEqual(references["coord_decErr"], original["coord_decErr"])
# One year difference
loader.applyProperMotions(references, self.epoch + 1.0*astropy.units.yr)
self.assertFloatsEqual(references["pm_raErr"], original["pm_raErr"])
self.assertFloatsEqual(references["pm_decErr"], original["pm_decErr"])
for orig, ref in zip(original, references):
self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
predictedRaErr = np.hypot(original["coord_raErr"], original["pm_raErr"])
predictedDecErr = np.hypot(original["coord_decErr"], original["pm_decErr"])
self.assertFloatsAlmostEqual(references["coord_raErr"], predictedRaErr)
self.assertFloatsAlmostEqual(references["coord_decErr"], predictedDecErr)
def testLoadPixelBox(self):
"""Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config."""
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
numFound = 0
for tupl, idList in self.compCats.items():
cent = ingestIndexTestBase.make_coord(*tupl)
bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary
ctr_pix = bbox.getCenter()
# catalog is sparse, so set pixel scale such that bbox encloses region
# used to generate compCats
pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth())
cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale)
wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix)
result = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a")
self.assertFalse("camFlux" in result.refCat.schema)
self.assertGreaterEqual(len(result.refCat), len(idList))
numFound += len(result.refCat)
self.assertGreater(numFound, 0)
def make_fake_refcat(center, flux, filterName):
"""Make a fake reference catalog."""
schema = LoadIndexedReferenceObjectsTask.makeMinimalSchema([filterName])
catalog = lsst.afw.table.SimpleCatalog(schema)
record = catalog.addNew()
record.setCoord(center)
record[filterName + '_flux'] = flux
record[filterName + '_fluxErr'] = flux*0.1
return catalog
def testLoadPixelBox(self):
"""Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config."""
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
numFound = 0
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary
ctr_pix = bbox.getCenter()
# catalog is sparse, so set pixel scale such that bbox encloses region
# used to generate compCats
pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth())
cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale)
wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix)
result = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a")
self.assertFalse("camFlux" in result.refCat.schema)
self.assertGreaterEqual(len(result.refCat), len(idList))
numFound += len(result.refCat)
self.assertGreater(numFound, 0)
def make_fake_refcat(center, flux, filterName):
"""Make a fake reference catalog."""
schema = LoadIndexedReferenceObjectsTask.makeMinimalSchema([filterName])
catalog = lsst.afw.table.SimpleCatalog(schema)
record = catalog.addNew()
record.setCoord(center)
record[filterName + '_flux'] = flux
record[filterName + '_fluxErr'] = flux * 0.1
return catalog
def testLoadSkyCircle(self):
"""Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config."""
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a')
self.assertTrue(lcat.refCat.isContiguous())
self.assertFalse("camFlux" in lcat.refCat.schema)
self.assertEqual(Counter(lcat.refCat['id']), Counter(idList))
if len(lcat.refCat) > 0:
# make sure there are no duplicate ids
self.assertEqual(len(set(Counter(lcat.refCat['id']).values())), 1)
self.assertEqual(len(set(Counter(idList).values())), 1)
for suffix in ("x", "y"):
self.assertTrue(np.all(np.isnan(lcat.refCat["centroid_%s" % (suffix,)])))
self.assertFalse(np.any(lcat.refCat["hasCentroid"]))
else:
self.assertEqual(len(idList), 0)
def testLoadSkyCircle(self):
"""Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config."""
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a')
self.assertTrue(lcat.refCat.isContiguous())
self.assertFalse("camFlux" in lcat.refCat.schema)
self.assertEqual(Counter(lcat.refCat['id']), Counter(idList))
if len(lcat.refCat) > 0:
# make sure there are no duplicate ids
self.assertEqual(len(set(Counter(lcat.refCat['id']).values())), 1)
self.assertEqual(len(set(Counter(idList).values())), 1)
# A default-loaded sky circle should not have centroids
self.assertNotIn("centroid_x", lcat.refCat.schema)
self.assertNotIn("centroid_y", lcat.refCat.schema)
self.assertNotIn("hasCentroid", lcat.refCat.schema)
else:
self.assertEqual(len(idList), 0)
def runAstrometry(self, butler, exp, icSrc):
refObjLoaderConfig = LoadIndexedReferenceObjectsTask.ConfigClass()
refObjLoaderConfig.ref_dataset_name = 'gaia_dr2_20191105'
refObjLoaderConfig.pixelMargin = 1000
refObjLoader = LoadIndexedReferenceObjectsTask(
butler=butler, config=refObjLoaderConfig)
astromConfig = AstrometryTask.ConfigClass()
astromConfig.wcsFitter.retarget(FitAffineWcsTask)
astromConfig.referenceSelector.doMagLimit = True
magLimit = MagnitudeLimit()
magLimit.minimum = 1
magLimit.maximum = 15
astromConfig.referenceSelector.magLimit = magLimit
astromConfig.referenceSelector.magLimit.fluxField = "phot_g_mean_flux"
astromConfig.matcher.maxRotationDeg = 5.99
astromConfig.matcher.maxOffsetPix = 3000
astromConfig.sourceSelector['matcher'].minSnr = 10
solver = AstrometryTask(config=astromConfig, refObjLoader=refObjLoader)
# TODO: Change this to doing this the proper way
referenceFilterName = self.config.referenceFilterOverride
referenceFilterLabel = afwImage.FilterLabel(
physical=referenceFilterName, band=referenceFilterName)
originalFilterLabel = exp.getFilterLabel(
) # there's a better way of doing this with the task I think
exp.setFilterLabel(referenceFilterLabel)
try:
astromResult = solver.run(sourceCat=icSrc, exposure=exp)
exp.setFilterLabel(originalFilterLabel)
except (RuntimeError, TaskError):
self.log.warn("Solver failed to run completely")
exp.setFilterLabel(originalFilterLabel)
return None
scatter = astromResult.scatterOnSky.asArcseconds()
if scatter < 1:
return astromResult
else:
self.log.warn("Failed to find an acceptable match")
return None
def testIngestConfigOverrides(self):
"""Test IngestIndexedReferenceTask with different configs.
"""
config2 = makeIngestIndexConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True,
withParallax=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'parallax', 'parallax_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
def main():
import argparse
class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
argparse.RawDescriptionHelpFormatter):
pass
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=CustomFormatter)
parser.add_argument(
"refCatPath",
help="Butler repo (written by IngestIndexedReferenceTask) containing the"
" reference catalogs to test.")
parser.add_argument(
"inputGlob",
help=
"A glob pattern specifying the files (read by IngestIndexedReferenceTask) to "
" check against the refCatPath output. (e.g. '/datasets/foo/*.csv'")
parser.add_argument(
"--nFiles",
default=5,
type=int,
help="Number of input files to test (randomly selected from inputGlob)."
)
parser.add_argument(
"--nPerFile",
default=100,
type=int,
help="Number of objects to test per file (randomly selected).")
parser.add_argument(
"--config",
help=
"A IngestIndexedReferenceConfig config file, for the field name mappings."
)
parser.add_argument(
"--ref_name",
help="The name of the reference catalog stored in refCatPath.")
args = parser.parse_args()
ingestConfig = IngestIndexedReferenceConfig()
ingestConfig.load(args.config)
butler = lsst.daf.persistence.Butler(args.refCatPath)
refObjConfig = LoadIndexedReferenceObjectsConfig()
refObjConfig.ref_dataset_name = args.ref_name
refObjLoader = LoadIndexedReferenceObjectsTask(butler, config=refObjConfig)
reader = ingestConfig.file_reader.target()
files = glob.glob(args.inputGlob)
files.sort()
for filename in random.sample(files, args.nFiles):
do_one_file(filename, refObjLoader, reader, ingestConfig,
args.nPerFile)
def testProperMotion(self):
"""Test proper motion correction"""
center = make_coord(93.0, -90.0)
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
references = loader.loadSkyCircle(center, self.searchRadius, filterName='a').refCat
original = references.copy(True)
# Zero epoch change --> no proper motion correction (except minor numerical effects)
loader.applyProperMotions(references, self.epoch)
self.assertFloatsAlmostEqual(references["coord_ra"], original["coord_ra"], rtol=1.0e-14)
self.assertFloatsAlmostEqual(references["coord_dec"], original["coord_dec"], rtol=1.0e-14)
self.assertFloatsEqual(references["coord_raErr"], original["coord_raErr"])
self.assertFloatsEqual(references["coord_decErr"], original["coord_decErr"])
# One year difference
loader.applyProperMotions(references, self.epoch + 1.0*astropy.units.yr)
self.assertFloatsEqual(references["pm_raErr"], original["pm_raErr"])
self.assertFloatsEqual(references["pm_decErr"], original["pm_decErr"])
for orig, ref in zip(original, references):
self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
predictedRaErr = np.hypot(original["coord_raErr"], original["pm_raErr"])
predictedDecErr = np.hypot(original["coord_decErr"], original["pm_decErr"])
self.assertFloatsAlmostEqual(references["coord_raErr"], predictedRaErr)
self.assertFloatsAlmostEqual(references["coord_decErr"], predictedDecErr)
def setUp(self):
refCatDir = os.path.join(os.path.dirname(__file__), "data", "sdssrefcat")
self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Extent2I(3001, 3001))
crpix = lsst.geom.Box2D(self.bbox).getCenter()
self.tanWcs = afwGeom.makeSkyWcs(crpix=crpix,
crval=lsst.geom.SpherePoint(215.5, 53.0, lsst.geom.degrees),
cdMatrix=afwGeom.makeCdMatrix(scale=5.1e-5*lsst.geom.degrees))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(self.tanWcs)
self.exposure.setFilter(afwImage.Filter("r", True))
butler = Butler(refCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def testLoadSkyCircle(self):
"""Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config."""
loader = LoadIndexedReferenceObjectsTask(butler=self.test_butler)
for tupl, idList in self.comp_cats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent,
self.search_radius,
filterName='a')
self.assertFalse("camFlux" in lcat.refCat.schema)
self.assertEqual(Counter(lcat.refCat['id']), Counter(idList))
if len(lcat.refCat) > 0:
# make sure there are no duplicate ids
self.assertEqual(len(set(Counter(lcat.refCat['id']).values())),
1)
self.assertEqual(len(set(Counter(idList).values())), 1)
for suffix in ("x", "y"):
self.assertTrue(
np.all(
np.isnan(lcat.refCat["centroid_%s" % (suffix, )])))
self.assertFalse(np.any(lcat.refCat["hasCentroid"]))
else:
self.assertEqual(len(idList), 0)
def make_fake_refcat(center, flux, filterName):
"""Make a fake reference catalog."""
schema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
[filterName], addProperMotion=True)
catalog = lsst.afw.table.SimpleCatalog(schema)
record = catalog.addNew()
record.setCoord(center)
record[filterName + '_flux'] = flux
record[filterName + '_fluxErr'] = flux * 0.1
record['pm_ra'] = lsst.geom.Angle(1)
record['pm_dec'] = lsst.geom.Angle(2)
record['epoch'] = 65432.1
return catalog
def runTest(withRaDecErr):
outputPath = os.path.join(self.outPath, "output_setsVersion"
+ "_withRaDecErr" if withRaDecErr else "")
# Test with multiple files and standard config
config = makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outputPath, self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath))
self.assertEqual(loader.dataset_config.format_version, 1)
def testIngestSetsVersion(self):
"""Test that newly ingested catalogs get the correct version number set.
"""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", self.outPath + "/output_setsVersion",
self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(
self.outPath + "/output_setsVersion"))
self.assertEqual(loader.dataset_config.format_version, 1)
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcCat = afwTable.SourceCatalog.readFits(
os.path.join(testDir, "data", "v695833-e0-c000.xy.fits"))
self.srcCat["slot_ApFlux_fluxSigma"] = 1
self.srcCat["slot_PsfFlux_fluxSigma"] = 1
# The .xy.fits file has sources in the range ~ [0,2000],[0,4500]
# which is bigger than the exposure
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(2048, 4612))
smallExposure = afwImage.ExposureF(
os.path.join(testDir, "data", "v695833-e0-c000-a00.sci.fits"))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
self.exposure.setCalib(smallExposure.getCalib())
coordKey = self.srcCat.getCoordKey()
centroidKey = self.srcCat.getCentroidKey()
wcs = self.exposure.getWcs()
for src in self.srcCat:
src.set(coordKey, wcs.pixelToSky(src.get(centroidKey)))
# Make a reference loader
butler = Butler(RefCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
logLevel = Log.TRACE
self.log = Log.getLogger('testPhotoCal')
self.log.setLevel(logLevel)
self.config = PhotoCalConfig()
self.config.match.matchRadius = 0.5
self.config.match.referenceSelection.doMagLimit = True
self.config.match.referenceSelection.magLimit.maximum = 22.0
self.config.match.referenceSelection.magLimit.fluxField = "i_flux"
self.config.match.referenceSelection.doFlags = True
self.config.match.referenceSelection.flags.good = ['photometric']
self.config.match.referenceSelection.flags.bad = ['resolved']
self.config.match.sourceSelection.doUnresolved = False # Don't have star/galaxy in the srcCat
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
self.config.fluxField = "base_PsfFlux_flux"
def _formatCatalog(self, fgcmStarCat, offsets):
"""
Turn an FGCM-formatted star catalog, applying zeropoint offsets.
Parameters
----------
fgcmStarCat: `afwTable.SimpleCatalog`
SimpleCatalog as output by fgcmcal
offsets: `list` with len(self.bands) entries
Zeropoint offsets to apply
Returns
-------
formattedCat: `afwTable.SimpleCatalog`
SimpleCatalog suitable for using as a reference catalog
"""
sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema)
minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(
self.bands, addCentroid=False, addIsResolved=True, coordErrDim=0)
sourceMapper.addMinimalSchema(minSchema)
for band in self.bands:
sourceMapper.editOutputSchema().addField('%s_nGood' % (band),
type=np.int32)
formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema())
formattedCat.reserve(len(fgcmStarCat))
formattedCat.extend(fgcmStarCat, mapper=sourceMapper)
# Note that we don't have to set `resolved` because the default is False
for b, band in enumerate(self.bands):
mag = fgcmStarCat['mag_std_noabs'][:, b] + offsets[b]
# We want fluxes in Jy from calibrated AB magnitudes
# (after applying offset)
# TODO: Full implementation of RFC-549 will have all reference
# catalogs in nJy instead of Jy.
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(
fgcmStarCat['magErr_std'][:, b], mag)
formattedCat['%s_flux' % (band)][:] = flux
formattedCat['%s_fluxErr' % (band)][:] = fluxErr
formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b]
return formattedCat
class TestAstrometricSolver(lsst.utils.tests.TestCase):
def setUp(self):
refCatDir = os.path.join(os.path.dirname(__file__), "data", "sdssrefcat")
self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Extent2I(3001, 3001))
crpix = lsst.geom.Box2D(self.bbox).getCenter()
self.tanWcs = afwGeom.makeSkyWcs(crpix=crpix,
crval=lsst.geom.SpherePoint(215.5, 53.0, lsst.geom.degrees),
cdMatrix=afwGeom.makeCdMatrix(scale=5.1e-5*lsst.geom.degrees))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(self.tanWcs)
self.exposure.setFilter(afwImage.Filter("r", True))
butler = Butler(refCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def tearDown(self):
del self.tanWcs
del self.exposure
del self.refObjLoader
def testTrivial(self):
"""Test fit with no distortion
"""
self.doTest(afwGeom.makeIdentityTransform())
def testRadial(self):
"""Test fit with radial distortion
The offset comes from the fact that the CCD is not centered
"""
self.doTest(afwGeom.makeRadialTransform([0, 1.01, 1e-7]))
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
self.exposure.setWcs(self.tanWcs)
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox, wcs=self.tanWcs, filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(schema=sourceSchema) # expand the schema
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader, schema=sourceSchema)
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCat.reserve(len(refCat))
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey)/100)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometry_used'):
count += 1
self.assertEqual(count, len(results.matches))
def doTest(self, pixelsToTanPixels, order=3):
"""Test using pixelsToTanPixels to distort the source positions
"""
distortedWcs = afwGeom.makeModifiedWcs(pixelTransform=pixelsToTanPixels, wcs=self.tanWcs,
modifyActualPixels=False)
self.exposure.setWcs(distortedWcs)
sourceCat = self.makeSourceCat(distortedWcs)
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = order
config.wcsFitter.numRejIter = 0
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
fitWcs = self.exposure.getWcs()
self.assertRaises(Exception, self.assertWcsAlmostEqualOverBBox, fitWcs, distortedWcs)
self.assertWcsAlmostEqualOverBBox(distortedWcs, fitWcs, self.bbox,
maxDiffSky=0.01*lsst.geom.arcseconds, maxDiffPix=0.02)
srcCoordKey = afwTable.CoordKey(sourceCat.schema["coord"])
refCoordKey = afwTable.CoordKey(results.refCat.schema["coord"])
refCentroidKey = afwTable.Point2DKey(results.refCat.schema["centroid"])
maxAngSep = 0*lsst.geom.radians
maxPixSep = 0
for refObj, src, d in results.matches:
refCoord = refObj.get(refCoordKey)
refPixPos = refObj.get(refCentroidKey)
srcCoord = src.get(srcCoordKey)
srcPixPos = src.getCentroid()
angSep = refCoord.separation(srcCoord)
maxAngSep = max(maxAngSep, angSep)
pixSep = math.hypot(*(srcPixPos-refPixPos))
maxPixSep = max(maxPixSep, pixSep)
print("max angular separation = %0.4f arcsec" % (maxAngSep.asArcseconds(),))
print("max pixel separation = %0.3f" % (maxPixSep,))
self.assertLess(maxAngSep.asArcseconds(), 0.0038)
self.assertLess(maxPixSep, 0.021)
# try again, invoking the reference selector
config.referenceSelector.doUnresolved = True
config.referenceSelector.unresolved.name = 'resolved'
solverRefSelect = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsRefSelect = solverRefSelect.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertLess(len(resultsRefSelect.matches), len(results.matches))
# try again, but without fitting the WCS, no reference selector
config.referenceSelector.doUnresolved = False
config.forceKnownWcs = True
solverNoFit = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsNoFit = solverNoFit.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertIsNone(resultsNoFit.scatterOnSky)
# fitting should result in matches that are at least as good
# (strictly speaking fitting might result in a larger match list with
# some outliers, but in practice this test passes)
meanFitDist = np.mean([match.distance for match in results.matches])
meanNoFitDist = np.mean([match.distance for match in resultsNoFit.matches])
self.assertLessEqual(meanFitDist, meanNoFitDist)
# try once again, without fitting the WCS, with the reference selector
# (this goes through a different code path)
config.referenceSelector.doUnresolved = True
solverNoFitRefSelect = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
resultsNoFitRefSelect = solverNoFitRefSelect.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertLess(len(resultsNoFitRefSelect.matches), len(resultsNoFit.matches))
def makeSourceCat(self, distortedWcs):
"""Make a source catalog by reading the position reference stars and distorting the positions
"""
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox, wcs=distortedWcs, filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(schema=sourceSchema) # expand the schema
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
sourceCat.reserve(len(refCat))
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey)/100)
return sourceCat
class TestAstrometricSolver(lsst.utils.tests.TestCase):
def setUp(self):
refCatDir = os.path.join(os.path.dirname(__file__), "data",
"sdssrefcat")
self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Extent2I(3001, 3001))
crpix = lsst.geom.Box2D(self.bbox).getCenter()
self.tanWcs = afwGeom.makeSkyWcs(
crpix=crpix,
crval=lsst.geom.SpherePoint(215.5, 53.0, lsst.geom.degrees),
cdMatrix=afwGeom.makeCdMatrix(scale=5.1e-5 * lsst.geom.degrees))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(self.tanWcs)
self.exposure.setFilter(afwImage.Filter("r", True))
butler = Butler(refCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def tearDown(self):
del self.tanWcs
del self.exposure
del self.refObjLoader
def testTrivial(self):
"""Test fit with no distortion
"""
self.doTest(afwGeom.makeIdentityTransform())
def testRadial(self):
"""Test fit with radial distortion
The offset comes from the fact that the CCD is not centered
"""
self.doTest(afwGeom.makeRadialTransform([0, 1.01, 1e-7]))
def testUsedFlag(self):
"""Test that the solver will record number of sources used to table
if it is passed a schema on initialization.
"""
self.exposure.setWcs(self.tanWcs)
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=self.tanWcs,
filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = 2
config.wcsFitter.numRejIter = 0
# schema must be passed to the solver task constructor
solver = AstrometryTask(config=config,
refObjLoader=self.refObjLoader,
schema=sourceSchema)
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCat.reserve(len(refCat))
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey) / 100)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
# check that the used flag is set the right number of times
count = 0
for source in sourceCat:
if source.get('calib_astrometry_used'):
count += 1
self.assertEqual(count, len(results.matches))
def doTest(self, pixelsToTanPixels, order=3):
"""Test using pixelsToTanPixels to distort the source positions
"""
distortedWcs = afwGeom.makeModifiedWcs(
pixelTransform=pixelsToTanPixels,
wcs=self.tanWcs,
modifyActualPixels=False)
self.exposure.setWcs(distortedWcs)
sourceCat = self.makeSourceCat(distortedWcs)
config = AstrometryTask.ConfigClass()
config.wcsFitter.order = order
config.wcsFitter.numRejIter = 0
solver = AstrometryTask(config=config, refObjLoader=self.refObjLoader)
results = solver.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
fitWcs = self.exposure.getWcs()
self.assertRaises(Exception, self.assertWcsAlmostEqualOverBBox, fitWcs,
distortedWcs)
self.assertWcsAlmostEqualOverBBox(distortedWcs,
fitWcs,
self.bbox,
maxDiffSky=0.01 *
lsst.geom.arcseconds,
maxDiffPix=0.02)
srcCoordKey = afwTable.CoordKey(sourceCat.schema["coord"])
refCoordKey = afwTable.CoordKey(results.refCat.schema["coord"])
refCentroidKey = afwTable.Point2DKey(results.refCat.schema["centroid"])
maxAngSep = 0 * lsst.geom.radians
maxPixSep = 0
for refObj, src, d in results.matches:
refCoord = refObj.get(refCoordKey)
refPixPos = refObj.get(refCentroidKey)
srcCoord = src.get(srcCoordKey)
srcPixPos = src.getCentroid()
angSep = refCoord.separation(srcCoord)
maxAngSep = max(maxAngSep, angSep)
pixSep = math.hypot(*(srcPixPos - refPixPos))
maxPixSep = max(maxPixSep, pixSep)
print("max angular separation = %0.4f arcsec" %
(maxAngSep.asArcseconds(), ))
print("max pixel separation = %0.3f" % (maxPixSep, ))
self.assertLess(maxAngSep.asArcseconds(), 0.0038)
self.assertLess(maxPixSep, 0.021)
# try again, invoking the reference selector
config.referenceSelector.doUnresolved = True
config.referenceSelector.unresolved.name = 'resolved'
solverRefSelect = AstrometryTask(config=config,
refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsRefSelect = solverRefSelect.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertLess(len(resultsRefSelect.matches), len(results.matches))
# try again, but without fitting the WCS, no reference selector
config.referenceSelector.doUnresolved = False
config.forceKnownWcs = True
solverNoFit = AstrometryTask(config=config,
refObjLoader=self.refObjLoader)
self.exposure.setWcs(distortedWcs)
resultsNoFit = solverNoFit.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertIsNone(resultsNoFit.scatterOnSky)
# fitting should result in matches that are at least as good
# (strictly speaking fitting might result in a larger match list with
# some outliers, but in practice this test passes)
meanFitDist = np.mean([match.distance for match in results.matches])
meanNoFitDist = np.mean(
[match.distance for match in resultsNoFit.matches])
self.assertLessEqual(meanFitDist, meanNoFitDist)
# try once again, without fitting the WCS, with the reference selector
# (this goes through a different code path)
config.referenceSelector.doUnresolved = True
solverNoFitRefSelect = AstrometryTask(config=config,
refObjLoader=self.refObjLoader)
resultsNoFitRefSelect = solverNoFitRefSelect.run(
sourceCat=sourceCat,
exposure=self.exposure,
)
self.assertLess(len(resultsNoFitRefSelect.matches),
len(resultsNoFit.matches))
def makeSourceCat(self, distortedWcs):
"""Make a source catalog by reading the position reference stars and distorting the positions
"""
loadRes = self.refObjLoader.loadPixelBox(bbox=self.bbox,
wcs=distortedWcs,
filterName="r")
refCat = loadRes.refCat
refCentroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
refFluxRKey = refCat.schema["r_flux"].asKey()
sourceSchema = afwTable.SourceTable.makeMinimalSchema()
measBase.SingleFrameMeasurementTask(
schema=sourceSchema) # expand the schema
sourceCat = afwTable.SourceCatalog(sourceSchema)
sourceCentroidKey = afwTable.Point2DKey(sourceSchema["slot_Centroid"])
sourceInstFluxKey = sourceSchema["slot_ApFlux_instFlux"].asKey()
sourceInstFluxErrKey = sourceSchema["slot_ApFlux_instFluxErr"].asKey()
sourceCat.reserve(len(refCat))
for refObj in refCat:
src = sourceCat.addNew()
src.set(sourceCentroidKey, refObj.get(refCentroidKey))
src.set(sourceInstFluxKey, refObj.get(refFluxRKey))
src.set(sourceInstFluxErrKey, refObj.get(refFluxRKey) / 100)
return sourceCat
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
default_config = IngestIndexedReferenceTask.ConfigClass()
# test ingest with default config
# This should raise since I haven't specified the ra/dec/mag columns.
with self.assertRaises(ValueError):
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output",
self.sky_catalog_file
],
config=default_config)
# test with ~minimum config. Mag errors are not technically necessary, but might as well test here
default_config.ra_name = 'ra_icrs'
default_config.dec_name = 'dec_icrs'
default_config.mag_column_list = ['a', 'b']
default_config.mag_err_column_map = {'a': 'a_err'}
# should raise since all columns need an error column if any do
with self.assertRaises(ValueError):
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output",
self.sky_catalog_file
],
config=default_config)
# test with multiple files and correct config
default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output_multifile",
self.sky_catalog_file, self.sky_catalog_file
],
config=default_config)
# test with config overrides
default_config = IngestIndexedReferenceTask.ConfigClass()
default_config.ra_name = 'ra'
default_config.dec_name = 'dec'
default_config.mag_column_list = ['a', 'b']
default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
default_config.dataset_config.ref_dataset_name = 'myrefcat'
default_config.dataset_config.indexer.active.depth = 10
default_config.is_photometric_name = 'is_phot'
default_config.is_resolved_name = 'is_res'
default_config.is_variable_name = 'is_var'
default_config.id_name = 'id'
default_config.extra_col_names = ['val1', 'val2', 'val3']
default_config.file_reader.header_lines = 1
default_config.file_reader.colnames = [
'id', 'ra', 'dec', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res',
'is_var', 'val1', 'val2', 'val3'
]
default_config.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output_override",
self.sky_catalog_file_delim
],
config=default_config)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.out_path + "/output_override")
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config)
cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a')
self.assertTrue(len(cat) > 0)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.test_repo_path)
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = self.test_dataset_name
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config)
cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a')
self.assertTrue(len(cat) > 0)