def testValidateRaDecMag(self):
config = makeIngestIndexConfig()
config.validate()
for name in ("ra_name", "dec_name", "mag_column_list"):
with self.subTest(name=name):
config = makeIngestIndexConfig()
setattr(config, name, None)
with self.assertRaises(ValueError):
config.validate()
def testValidateParallax(self):
"""Validation should fail if any parallax-related fields are missing.
"""
names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"]
config = makeIngestIndexConfig(withParallax=True)
config.validate()
del config
for name in names:
with self.subTest(name=name):
config = makeIngestIndexConfig(withParallax=True)
setattr(config, name, None)
with self.assertRaises(ValueError, msg=name):
config.validate()
def setUp(self):
np.random.seed(10)
self.log = lsst.log.Log.getLogger("TestIngestIndexManager")
self.config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=True)
self.config.id_name = 'id'
depth = 2 # very small depth, for as few pixels as possible.
self.indexer = HtmIndexer(depth)
self.htm = lsst.sphgeom.HtmPixelization(depth)
ingester = IngestIndexedReferenceTask(self.config)
dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
('a', '<f8'), ('a_err', '<f8')]
self.schema, self.key_map = ingester.makeSchema(dtype)
self.fileReader = ReadTextCatalogTask()
self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
self.matchedPixels = np.array([1, 1, 2, 2, 3])
self.path = tempfile.mkdtemp()
self.filenames = {x: os.path.join(self.path, "%d.fits" % x) for x in set(self.matchedPixels)}
self.worker = IngestIndexManager(self.filenames,
self.config,
self.fileReader,
self.indexer,
self.schema,
self.key_map,
self.htm.universe()[0],
addRefCatMetadata,
self.log)
def runTest(withRaDecErr):
# Generate a second catalog, with different ids
inPath1 = tempfile.mkdtemp()
skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
inPath2 = tempfile.mkdtemp()
skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
# override some field names, and use multiple cores
config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# use a very small HTM pixelization depth to ensure there will be collisions when
# ingesting the files in parallel
config.dataset_config.indexer.active.depth = 2
# np.savetxt prepends '# ' to the header lines, so use a reader that understands that
config.file_reader.format = 'ascii.commented_header'
config.n_processes = 2 # use multiple cores for this test only
config.id_name = 'id' # Use the ids from the generated catalogs
outpath = os.path.join(self.outPath, "output_multifile_parallel",
"_withRaDecErr" if withRaDecErr else "")
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outpath,
skyCatalogFile1, skyCatalogFile2], config=config)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(outpath)
loaderConfig = LoadIndexedReferenceObjectsConfig()
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, skyCatalog1, config)
self.checkAllRowsInRefcat(loader, skyCatalog2, config)
def testValidatePm(self):
basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"]
for withPmErr in (False, True):
config = makeIngestIndexConfig(withPm=True, withPmErr=withPmErr)
config.validate()
del config
if withPmErr:
names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"]
else:
names = basicNames
for name in names:
with self.subTest(name=name, withPmErr=withPmErr):
config = makeIngestIndexConfig(withPm=True, withPmErr=withPmErr)
setattr(config, name, None)
with self.assertRaises(ValueError):
config.validate()
def testValidateRaDecErr(self):
# check that a basic config validates
config = makeIngestIndexConfig(withRaDecErr=True)
config.validate()
# check that a config with any of these fields missing does not validate
for name in ("ra_err_name", "dec_err_name", "coord_err_unit"):
with self.subTest(name=name):
config = makeIngestIndexConfig(withRaDecErr=True)
setattr(config, name, None)
with self.assertRaises(ValueError):
config.validate()
# check that coord_err_unit must be an astropy unit
config = makeIngestIndexConfig(withRaDecErr=True)
config.coord_err_unit = "nonsense unit"
with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"):
config.validate()
def testValidateMagErr(self):
config = makeIngestIndexConfig(withMagErr=True)
config.validate()
# test for missing names
for name in config.mag_column_list:
with self.subTest(name=name):
config = makeIngestIndexConfig(withMagErr=True)
del config.mag_err_column_map[name]
with self.assertRaises(ValueError):
config.validate()
# test for incorrect names
for name in config.mag_column_list:
with self.subTest(name=name):
config = makeIngestIndexConfig(withMagErr=True)
config.mag_err_column_map["badName"] = config.mag_err_column_map[name]
del config.mag_err_column_map[name]
with self.assertRaises(ValueError):
config.validate()
def runTest(withRaDecErr):
outputPath = os.path.join(self.outPath, "output_setsVersion"
+ "_withRaDecErr" if withRaDecErr else "")
# Test with multiple files and standard config
config = makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outputPath, self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath))
self.assertEqual(loader.dataset_config.format_version, 1)
def testIngestConfigOverrides(self):
"""Test IngestIndexedReferenceTask with different configs.
"""
config2 = makeIngestIndexConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True,
withParallax=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'parallax', 'parallax_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)