def runTest(withRaDecErr):
# Generate a second catalog, with different ids
inPath1 = tempfile.mkdtemp()
skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
inPath2 = tempfile.mkdtemp()
skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
# override some field names, and use multiple cores
config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# use a very small HTM pixelization depth to ensure there will be collisions when
# ingesting the files in parallel
config.dataset_config.indexer.active.depth = 2
# np.savetxt prepends '# ' to the header lines, so use a reader that understands that
config.file_reader.format = 'ascii.commented_header'
config.n_processes = 2 # use multiple cores for this test only
config.id_name = 'id' # Use the ids from the generated catalogs
outpath = os.path.join(self.outPath, "output_multifile_parallel",
"_withRaDecErr" if withRaDecErr else "")
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outpath,
skyCatalogFile1, skyCatalogFile2], config=config)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(outpath)
loaderConfig = LoadIndexedReferenceObjectsConfig()
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, skyCatalog1, config)
self.checkAllRowsInRefcat(loader, skyCatalog2, config)
def setUp(self):
np.random.seed(10)
self.log = lsst.log.Log.getLogger("TestIngestIndexManager")
self.config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=True)
self.config.id_name = 'id'
depth = 2 # very small depth, for as few pixels as possible.
self.indexer = HtmIndexer(depth)
self.htm = lsst.sphgeom.HtmPixelization(depth)
ingester = IngestIndexedReferenceTask(self.config)
dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
('a', '<f8'), ('a_err', '<f8')]
self.schema, self.key_map = ingester.makeSchema(dtype)
self.fileReader = ReadTextCatalogTask()
self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
self.matchedPixels = np.array([1, 1, 2, 2, 3])
self.path = tempfile.mkdtemp()
self.filenames = {x: os.path.join(self.path, "%d.fits" % x) for x in set(self.matchedPixels)}
self.worker = IngestIndexManager(self.filenames,
self.config,
self.fileReader,
self.indexer,
self.schema,
self.key_map,
self.htm.universe()[0],
addRefCatMetadata,
self.log)
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", self.outPath+"/output_multifile",
self.skyCatalogFile, self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(self.outPath+"/output_multifile"))
self.assertEqual(loader.dataset_config.format_version, 1)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
def setUpClass(cls):
cls.outPath = tempfile.mkdtemp()
cls.testCatPath = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "data",
"testHtmIndex.fits")
# arbitrary, but reasonable, amount of proper motion (angle/year)
# and direction of proper motion
cls.properMotionAmt = 3.0 * lsst.geom.arcseconds
cls.properMotionDir = 45 * lsst.geom.degrees
cls.properMotionErr = 1e-3 * lsst.geom.arcseconds
cls.epoch = astropy.time.Time(58206.861330339219,
scale="tai",
format="mjd")
ret = cls.make_skyCatalog(cls.outPath)
cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = ret
cls.testRas = [210., 14.5, 93., 180., 286., 0.]
cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.]
cls.searchRadius = 3. * lsst.geom.degrees
cls.compCats = {
} # dict of center coord: list of IDs of stars within cls.searchRadius of center
cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius
config = IndexerRegistry['HTM'].ConfigClass()
# Match on disk comparison file
config.depth = cls.depth
cls.indexer = IndexerRegistry['HTM'](config)
for ra in cls.testRas:
for dec in cls.testDecs:
tupl = (ra, dec)
cent = make_coord(*tupl)
cls.compCats[tupl] = []
for rec in cls.skyCatalog:
if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(
cent) < cls.searchRadius:
cls.compCats[tupl].append(rec['id'])
cls.testRepoPath = cls.outPath + "/test_repo"
config = cls.makeConfig(withMagErr=True,
withRaDecErr=True,
withPm=True,
withPmErr=True)
# To match on disk test data
config.dataset_config.indexer.active.depth = cls.depth
config.id_name = 'id'
config.pm_scale = 1000.0 # arcsec/yr --> mas/yr
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", cls.testRepoPath, cls.skyCatalogFile],
config=config)
cls.defaultDatasetName = config.dataset_config.ref_dataset_name
cls.testDatasetName = 'diff_ref_name'
cls.testButler = dafPersist.Butler(cls.testRepoPath)
os.symlink(
os.path.join(cls.testRepoPath, 'ref_cats', cls.defaultDatasetName),
os.path.join(cls.testRepoPath, 'ref_cats', cls.testDatasetName))
def runTest(withRaDecErr):
outputPath = os.path.join(self.outPath, "output_setsVersion"
+ "_withRaDecErr" if withRaDecErr else "")
# Test with multiple files and standard config
config = makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outputPath, self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath))
self.assertEqual(loader.dataset_config.format_version, 1)
def setUpClass(cls):
cls.out_path = tempfile.mkdtemp()
cls.test_cat_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "data",
"testHtmIndex.fits")
cls.test_cat = afwTable.SourceCatalog.readFits(cls.test_cat_path)
ret = cls.make_sky_catalog(cls.out_path)
cls.sky_catalog_file, cls.sky_catalog_file_delim, cls.sky_catalog = ret
cls.test_ras = [210., 14.5, 93., 180., 286., 0.]
cls.test_decs = [-90., -51., -30.1, 0., 27.3, 62., 90.]
cls.search_radius = 3. * afwGeom.degrees
cls.comp_cats = {
} # dict of center coord: list of IDs of stars within cls.search_radius of center
cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius
config = IndexerRegistry['HTM'].ConfigClass()
# Match on disk comparison file
config.depth = cls.depth
cls.indexer = IndexerRegistry['HTM'](config)
for ra in cls.test_ras:
for dec in cls.test_decs:
tupl = (ra, dec)
cent = make_coord(*tupl)
cls.comp_cats[tupl] = []
for rec in cls.sky_catalog:
if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(
cent) < cls.search_radius:
cls.comp_cats[tupl].append(rec['id'])
cls.test_repo_path = cls.out_path + "/test_repo"
config = IngestIndexedReferenceTask.ConfigClass()
# To match on disk test data
config.dataset_config.indexer.active.depth = cls.depth
config.ra_name = 'ra_icrs'
config.dec_name = 'dec_icrs'
config.mag_column_list = ['a', 'b']
config.id_name = 'id'
config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", cls.test_repo_path, cls.sky_catalog_file
],
config=config)
cls.default_dataset_name = config.dataset_config.ref_dataset_name
cls.test_dataset_name = 'diff_ref_name'
cls.test_butler = dafPersist.Butler(cls.test_repo_path)
os.symlink(
os.path.join(cls.test_repo_path, 'ref_cats',
cls.default_dataset_name),
os.path.join(cls.test_repo_path, 'ref_cats',
cls.test_dataset_name))
def testIngestSetsVersion(self):
"""Test that newly ingested catalogs get the correct version number set.
"""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", self.outPath + "/output_setsVersion",
self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(
self.outPath + "/output_setsVersion"))
self.assertEqual(loader.dataset_config.format_version, 1)
def testAgainstPersisted(self):
shardId = 2222
dataset_name = IngestIndexedReferenceTask.ConfigClass(
).dataset_config.ref_dataset_name
dataId = self.indexer.makeDataId(shardId, dataset_name)
self.assertTrue(self.testButler.datasetExists('ref_cat', dataId))
refCat = self.testButler.get('ref_cat', dataId)
if REGENERATE_COMPARISON:
if os.path.exists(self.testCatPath):
os.unlink(self.testCatPath)
refCat.writeFits(self.testCatPath)
self.fail(
"New comparison data written; unset REGENERATE_COMPARISON in order to proceed"
)
ex1 = refCat.extract('*')
testCat = afwTable.SimpleCatalog.readFits(self.testCatPath)
ex2 = testCat.extract('*')
self.assertEqual(set(ex1.keys()), set(ex2.keys()))
for kk in ex1:
np.testing.assert_array_almost_equal(
ex1[kk],
ex2[kk],
)
def setUpClass(cls):
cls.outPath = tempfile.mkdtemp()
cls.testCatPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data",
"testHtmIndex.fits")
# arbitrary, but reasonable, amount of proper motion (angle/year)
# and direction of proper motion
cls.properMotionAmt = 3.0*lsst.geom.arcseconds
cls.properMotionDir = 45*lsst.geom.degrees
cls.properMotionErr = 1e-3*lsst.geom.arcseconds
cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd")
ret = cls.make_skyCatalog(cls.outPath)
cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = ret
cls.testRas = [210., 14.5, 93., 180., 286., 0.]
cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.]
cls.searchRadius = 3. * lsst.geom.degrees
cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center
cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius
config = IndexerRegistry['HTM'].ConfigClass()
# Match on disk comparison file
config.depth = cls.depth
cls.indexer = IndexerRegistry['HTM'](config)
for ra in cls.testRas:
for dec in cls.testDecs:
tupl = (ra, dec)
cent = make_coord(*tupl)
cls.compCats[tupl] = []
for rec in cls.skyCatalog:
if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius:
cls.compCats[tupl].append(rec['id'])
cls.testRepoPath = cls.outPath+"/test_repo"
config = cls.makeConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True)
# To match on disk test data
config.dataset_config.indexer.active.depth = cls.depth
config.id_name = 'id'
config.pm_scale = 1000.0 # arcsec/yr --> mas/yr
IngestIndexedReferenceTask.parseAndRun(args=[INPUT_DIR, "--output", cls.testRepoPath,
cls.skyCatalogFile], config=config)
cls.defaultDatasetName = config.dataset_config.ref_dataset_name
cls.testDatasetName = 'diff_ref_name'
cls.testButler = dafPersist.Butler(cls.testRepoPath)
os.symlink(os.path.join(cls.testRepoPath, 'ref_cats', cls.defaultDatasetName),
os.path.join(cls.testRepoPath, 'ref_cats', cls.testDatasetName))
def testAgainstPersisted(self):
pix_id = 671901
dataset_name = IngestIndexedReferenceTask.ConfigClass(
).dataset_config.ref_dataset_name
data_id = self.indexer.make_data_id(pix_id, dataset_name)
self.assertTrue(self.test_butler.datasetExists('ref_cat', data_id))
ref_cat = self.test_butler.get('ref_cat', data_id)
ex1 = ref_cat.extract('*')
ex2 = self.test_cat.extract('*')
# compare sets as the order may be different
self.assertDictEqual(ex1, ex2)
def testIngestConfigOverrides(self):
"""Test IngestIndexedReferenceTask with different configs.
"""
config2 = makeIngestIndexConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True,
withParallax=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'parallax', 'parallax_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
def makeConfig(withMagErr=False,
withRaDecErr=False,
withPm=False,
withPmErr=False,
withParallax=False,
withParallaxErr=False):
"""Make a config for IngestIndexedReferenceTask
This is primarily intended to simplify tests of config validation,
so fields that are not validated are not set.
However, it can calso be used to reduce boilerplate in other tests.
"""
config = IngestIndexedReferenceTask.ConfigClass()
config.pm_scale = 1000.0
config.ra_name = 'ra_icrs'
config.dec_name = 'dec_icrs'
config.mag_column_list = ['a', 'b']
if withMagErr:
config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
if withRaDecErr:
config.ra_err_name = "ra_err"
config.dec_err_name = "dec_err"
if withPm:
config.pm_ra_name = "pm_ra"
config.pm_dec_name = "pm_dec"
if withPmErr:
config.pm_ra_err_name = "pm_ra_err"
config.pm_dec_err_name = "pm_dec_err"
if withParallax:
config.parallax_name = "parallax"
if withParallaxErr:
config.parallax_err_name = "parallax_err"
if withPm or withParallax:
config.epoch_name = "unixtime"
config.epoch_format = "unix"
config.epoch_scale = "utc"
return config
def testAgainstPersisted(self):
pix_id = 2222
dataset_name = IngestIndexedReferenceTask.ConfigClass(
).dataset_config.ref_dataset_name
data_id = self.indexer.make_data_id(pix_id, dataset_name)
self.assertTrue(self.test_butler.datasetExists('ref_cat', data_id))
ref_cat = self.test_butler.get('ref_cat', data_id)
if REGENERATE_COMPARISON:
os.unlink(self.test_cat_path)
ref_cat.writeFits(self.test_cat_path)
self.fail(
"New comparison data written; unset REGENERATE_COMPARISON in order to proceed"
)
ex1 = ref_cat.extract('*')
ex2 = self.test_cat.extract('*')
self.assertEqual(set(ex1.keys()), set(ex2.keys()))
for kk in ex1:
np.testing.assert_array_equal(ex1[kk], ex2[kk])
#!/usr/bin/env python from lsst.meas.algorithms import IngestIndexedReferenceTask IngestIndexedReferenceTask.parseAndRun()
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
IngestIndexedReferenceTask.parseAndRun(args=[
INPUT_DIR, "--output", self.outPath + "/output_multifile",
self.skyCatalogFile, self.skyCatalogFile
],
config=config)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id',
'ra',
'dec',
'ra_err',
'dec_err',
'a',
'a_err',
'b',
'b_err',
'is_phot',
'is_res',
'is_var',
'val1',
'val2',
'val3',
'pm_ra',
'pm_dec',
'pm_ra_err',
'pm_dec_err',
'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
INPUT_DIR, "--output", self.outPath + "/output_override",
self.skyCatalogFileDelim
],
config=config2)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath + "/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig)
cat = loader.loadSkyCircle(cent, self.searchRadius,
filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig2)
cat = loader.loadSkyCircle(cent, self.searchRadius,
filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
default_config = IngestIndexedReferenceTask.ConfigClass()
# test ingest with default config
# This should raise since I haven't specified the ra/dec/mag columns.
with self.assertRaises(ValueError):
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output",
self.sky_catalog_file
],
config=default_config)
# test with ~minimum config. Mag errors are not technically necessary, but might as well test here
default_config.ra_name = 'ra_icrs'
default_config.dec_name = 'dec_icrs'
default_config.mag_column_list = ['a', 'b']
default_config.mag_err_column_map = {'a': 'a_err'}
# should raise since all columns need an error column if any do
with self.assertRaises(ValueError):
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output",
self.sky_catalog_file
],
config=default_config)
# test with multiple files and correct config
default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output_multifile",
self.sky_catalog_file, self.sky_catalog_file
],
config=default_config)
# test with config overrides
default_config = IngestIndexedReferenceTask.ConfigClass()
default_config.ra_name = 'ra'
default_config.dec_name = 'dec'
default_config.mag_column_list = ['a', 'b']
default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
default_config.dataset_config.ref_dataset_name = 'myrefcat'
default_config.dataset_config.indexer.active.depth = 10
default_config.is_photometric_name = 'is_phot'
default_config.is_resolved_name = 'is_res'
default_config.is_variable_name = 'is_var'
default_config.id_name = 'id'
default_config.extra_col_names = ['val1', 'val2', 'val3']
default_config.file_reader.header_lines = 1
default_config.file_reader.colnames = [
'id', 'ra', 'dec', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res',
'is_var', 'val1', 'val2', 'val3'
]
default_config.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output_override",
self.sky_catalog_file_delim
],
config=default_config)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.out_path + "/output_override")
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config)
cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a')
self.assertTrue(len(cat) > 0)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.test_repo_path)
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = self.test_dataset_name
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config)
cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a')
self.assertTrue(len(cat) > 0)
def testIngest(self):
"""Test IngestIndexedReferenceTask with different configs."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(args=[
self.input_dir, "--output", self.outPath + "/output_multifile",
self.skyCatalogFile, self.skyCatalogFile
],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(
butler=dafPersist.Butler(self.outPath + "/output_multifile"))
self.assertEqual(loader.dataset_config.format_version, 1)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id',
'ra',
'dec',
'ra_err',
'dec_err',
'a',
'a_err',
'b',
'b_err',
'is_phot',
'is_res',
'is_var',
'val1',
'val2',
'val3',
'pm_ra',
'pm_dec',
'pm_ra_err',
'pm_dec_err',
'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
self.input_dir, "--output", self.outPath + "/output_override",
self.skyCatalogFileDelim
],
config=config2)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath + "/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig)
self.checkAllRowsInRefcat(loader, self.skyCatalog)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig2)
self.checkAllRowsInRefcat(loader, self.skyCatalog)
"%i", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g",
"%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g",
"%.15g", "%.15g", "%.15g", "%.15g"
])
np.savetxt(outPath + "/ref.txt", arr, delimiter=",", **saveKwargs)
return outPath + "/ref.txt"
inPath = os.path.join(lsst.utils.getPackageDir('meas_algorithms'),
'tests/data')
outPath = os.path.join(inPath, "version%s" % LATEST_FORMAT_VERSION)
skyCatalogFile = make_skyCatalog(inPath)
config = IngestIndexedReferenceTask.ConfigClass()
config.ra_name = 'ra_icrs'
config.dec_name = 'dec_icrs'
config.mag_column_list = ['a', 'b']
config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
config.dataset_config.indexer.active.depth = 4
OBS_TEST_DIR = lsst.utils.getPackageDir('obs_test')
obsTestPath = os.path.join(OBS_TEST_DIR, "data", "input")
IngestIndexedReferenceTask.parseAndRun(
args=[obsTestPath, "--output", outPath, skyCatalogFile], config=config)
# cleanup files and make the mapper independent of obs_test's path
os.remove(skyCatalogFile)
os.remove(os.path.join(outPath, 'repositoryCfg.yaml'))
with open(os.path.join(outPath, '_mapper'), 'w') as mapperFile: