def testIngest(self):
"""Test IngestIndexedReferenceTask."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", self.outPath+"/output_multifile",
self.skyCatalogFile, self.skyCatalogFile],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(self.outPath+"/output_multifile"))
self.assertEqual(loader.dataset_config.format_version, 1)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[INPUT_DIR, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
def main():
import argparse
class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
argparse.RawDescriptionHelpFormatter):
pass
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=CustomFormatter)
parser.add_argument(
"refCatPath",
help="Butler repo (written by IngestIndexedReferenceTask) containing the"
" reference catalogs to test.")
parser.add_argument(
"inputGlob",
help=
"A glob pattern specifying the files (read by IngestIndexedReferenceTask) to "
" check against the refCatPath output. (e.g. '/datasets/foo/*.csv'")
parser.add_argument(
"--nFiles",
default=5,
type=int,
help="Number of input files to test (randomly selected from inputGlob)."
)
parser.add_argument(
"--nPerFile",
default=100,
type=int,
help="Number of objects to test per file (randomly selected).")
parser.add_argument(
"--config",
help=
"A IngestIndexedReferenceConfig config file, for the field name mappings."
)
parser.add_argument(
"--ref_name",
help="The name of the reference catalog stored in refCatPath.")
args = parser.parse_args()
ingestConfig = IngestIndexedReferenceConfig()
ingestConfig.load(args.config)
butler = lsst.daf.persistence.Butler(args.refCatPath)
refObjConfig = LoadIndexedReferenceObjectsConfig()
refObjConfig.ref_dataset_name = args.ref_name
refObjLoader = LoadIndexedReferenceObjectsTask(butler, config=refObjConfig)
reader = ingestConfig.file_reader.target()
files = glob.glob(args.inputGlob)
files.sort()
for filename in random.sample(files, args.nFiles):
do_one_file(filename, refObjLoader, reader, ingestConfig,
args.nPerFile)
def runTest(withRaDecErr):
# Generate a second catalog, with different ids
inPath1 = tempfile.mkdtemp()
skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123)
inPath2 = tempfile.mkdtemp()
skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11)
# override some field names, and use multiple cores
config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True,
withPm=True, withPmErr=True)
# use a very small HTM pixelization depth to ensure there will be collisions when
# ingesting the files in parallel
config.dataset_config.indexer.active.depth = 2
# np.savetxt prepends '# ' to the header lines, so use a reader that understands that
config.file_reader.format = 'ascii.commented_header'
config.n_processes = 2 # use multiple cores for this test only
config.id_name = 'id' # Use the ids from the generated catalogs
outpath = os.path.join(self.outPath, "output_multifile_parallel",
"_withRaDecErr" if withRaDecErr else "")
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", outpath,
skyCatalogFile1, skyCatalogFile2], config=config)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(outpath)
loaderConfig = LoadIndexedReferenceObjectsConfig()
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, skyCatalog1, config)
self.checkAllRowsInRefcat(loader, skyCatalog2, config)
def testDefaultFilterAndFilterMap(self):
"""Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
config = LoadIndexedReferenceObjectsConfig()
config.defaultFilter = "b"
config.filterMap = {"aprime": "a"}
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius)
self.assertEqual(lcat.fluxField, "camFlux")
if len(idList) > 0:
defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
self.assertTrue(defFluxFieldName in lcat.refCat.schema)
aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
break # just need one test
def testDefaultFilterAndFilterMap(self):
"""Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
config = LoadIndexedReferenceObjectsConfig()
config.defaultFilter = "b"
config.filterMap = {"aprime": "a"}
loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
for tupl, idList in self.compCats.items():
cent = make_coord(*tupl)
lcat = loader.loadSkyCircle(cent, self.searchRadius)
self.assertEqual(lcat.fluxField, "camFlux")
if len(idList) > 0:
defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
self.assertTrue(defFluxFieldName in lcat.refCat.schema)
aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
break # just need one test
def testLoadIndexedReferenceConfig(self):
"""Make sure LoadIndexedReferenceConfig has needed fields."""
"""
Including at least one from the base class LoadReferenceObjectsConfig
"""
config = LoadIndexedReferenceObjectsConfig()
self.assertEqual(config.ref_dataset_name, "cal_ref_cat")
self.assertEqual(config.defaultFilter, "")
def testIngestConfigOverrides(self):
"""Test IngestIndexedReferenceTask with different configs.
"""
config2 = makeIngestIndexConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True,
withParallax=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot',
'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err',
'pm_dec_err', 'parallax', 'parallax_err', 'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(
args=[self.input_dir, "--output", self.outPath+"/output_override",
self.skyCatalogFileDelim], config=config2)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath+"/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2)
self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
default_config = IngestIndexedReferenceTask.ConfigClass()
# test ingest with default config
# This should raise since I haven't specified the ra/dec/mag columns.
with self.assertRaises(ValueError):
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output",
self.sky_catalog_file
],
config=default_config)
# test with ~minimum config. Mag errors are not technically necessary, but might as well test here
default_config.ra_name = 'ra_icrs'
default_config.dec_name = 'dec_icrs'
default_config.mag_column_list = ['a', 'b']
default_config.mag_err_column_map = {'a': 'a_err'}
# should raise since all columns need an error column if any do
with self.assertRaises(ValueError):
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output",
self.sky_catalog_file
],
config=default_config)
# test with multiple files and correct config
default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output_multifile",
self.sky_catalog_file, self.sky_catalog_file
],
config=default_config)
# test with config overrides
default_config = IngestIndexedReferenceTask.ConfigClass()
default_config.ra_name = 'ra'
default_config.dec_name = 'dec'
default_config.mag_column_list = ['a', 'b']
default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
default_config.dataset_config.ref_dataset_name = 'myrefcat'
default_config.dataset_config.indexer.active.depth = 10
default_config.is_photometric_name = 'is_phot'
default_config.is_resolved_name = 'is_res'
default_config.is_variable_name = 'is_var'
default_config.id_name = 'id'
default_config.extra_col_names = ['val1', 'val2', 'val3']
default_config.file_reader.header_lines = 1
default_config.file_reader.colnames = [
'id', 'ra', 'dec', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res',
'is_var', 'val1', 'val2', 'val3'
]
default_config.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
input_dir, "--output", self.out_path + "/output_override",
self.sky_catalog_file_delim
],
config=default_config)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.out_path + "/output_override")
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config)
cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a')
self.assertTrue(len(cat) > 0)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.test_repo_path)
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = self.test_dataset_name
loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config)
cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a')
self.assertTrue(len(cat) > 0)
def testIngest(self):
"""Test IngestIndexedReferenceTask with different configs."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
# don't use the default depth, to avoid taking the time to create thousands of file locks
config.dataset_config.indexer.active.depth = self.depth
IngestIndexedReferenceTask.parseAndRun(args=[
self.input_dir, "--output", self.outPath + "/output_multifile",
self.skyCatalogFile, self.skyCatalogFile
],
config=config)
# A newly-ingested refcat should be marked format_version=1.
loader = LoadIndexedReferenceObjectsTask(
butler=dafPersist.Butler(self.outPath + "/output_multifile"))
self.assertEqual(loader.dataset_config.format_version, 1)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id',
'ra',
'dec',
'ra_err',
'dec_err',
'a',
'a_err',
'b',
'b_err',
'is_phot',
'is_res',
'is_var',
'val1',
'val2',
'val3',
'pm_ra',
'pm_dec',
'pm_ra_err',
'pm_dec_err',
'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
self.input_dir, "--output", self.outPath + "/output_override",
self.skyCatalogFileDelim
],
config=config2)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath + "/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig)
self.checkAllRowsInRefcat(loader, self.skyCatalog)
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig2)
self.checkAllRowsInRefcat(loader, self.skyCatalog)
def _runFgcmOutputProducts(self, visitDataRefName, ccdDataRefName,
filterMapping, zpOffsets, testVisit, testCcd,
testFilter, testBandIndex):
"""
"""
if self.logLevel is not None:
self.otherArgs.extend(['--loglevel', 'fgcmcal=%s' % self.logLevel])
args = [self.inputDir, '--output', self.testDir, '--doraise']
args.extend(self.otherArgs)
result = fgcmcal.FgcmOutputProductsTask.parseAndRun(
args=args, config=self.config, doReturnResults=True)
self._checkResult(result)
# Extract the offsets from the results
offsets = result.resultList[0].results.offsets
self.assertFloatsAlmostEqual(offsets[0], zpOffsets[0], rtol=1e-6)
self.assertFloatsAlmostEqual(offsets[1], zpOffsets[1], rtol=1e-6)
butler = dafPersistence.butler.Butler(self.testDir)
# Test the reference catalog stars
# Read in the raw stars...
rawStars = butler.get('fgcmStandardStars', fgcmcycle=0)
# Read in the new reference catalog...
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = 'fgcm_stars'
task = LoadIndexedReferenceObjectsTask(butler, config=config)
# Read in a giant radius to get them all
refStruct = task.loadSkyCircle(rawStars[0].getCoord(),
5.0 * lsst.geom.degrees,
filterName='r')
# Make sure all the stars are there
self.assertEqual(len(rawStars), len(refStruct.refCat))
# And make sure the numbers are consistent
test, = np.where(rawStars['id'][0] == refStruct.refCat['id'])
mag = rawStars['mag_std_noabs'][0, 0] + offsets[0]
flux = afwImage.fluxFromABMag(mag)
fluxErr = afwImage.fluxErrFromABMagErr(rawStars['magerr_std'][0, 0],
mag)
self.assertFloatsAlmostEqual(flux,
refStruct.refCat['r_flux'][test[0]],
rtol=1e-6)
self.assertFloatsAlmostEqual(fluxErr,
refStruct.refCat['r_fluxErr'][test[0]],
rtol=1e-6)
# Test the joincal_photoCalib output
zptCat = butler.get('fgcmZeropoints', fgcmcycle=0)
selected = (zptCat['fgcmflag'] < 16)
# Read in all the calibrations, these should all be there
for rec in zptCat[selected]:
testCal = butler.get('jointcal_photoCalib',
dataId={
visitDataRefName: int(rec['visit']),
ccdDataRefName: int(rec['ccd']),
'filter':
filterMapping[rec['filtername']],
'tract': 0
})
# Our round-trip tests will be on this final one which is still loaded
testCal = butler.get('jointcal_photoCalib',
dataId={
visitDataRefName: int(testVisit),
ccdDataRefName: int(testCcd),
'filter': filterMapping[testFilter],
'tract': 0
})
src = butler.get('src',
dataId={
visitDataRefName: int(testVisit),
ccdDataRefName: int(testCcd)
})
# Only test sources with positive flux
gdSrc = (src['slot_CalibFlux_flux'] > 0.0)
# We need to apply the calibration offset to the fgcmzpt (which is internal
# and doesn't know about that yet)
testZpInd, = np.where((zptCat['visit'] == testVisit)
& (zptCat['ccd'] == testCcd))
fgcmZpt = zptCat['fgcmzpt'][testZpInd] + offsets[testBandIndex]
# This is the magnitude through the mean calibration
photoCalMeanCalMags = np.zeros(gdSrc.sum())
# This is the magnitude through the full focal-plane variable mags
photoCalMags = np.zeros_like(photoCalMeanCalMags)
# This is the magnitude with the FGCM (central-ccd) zeropoint
zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
for i, rec in enumerate(src[gdSrc]):
photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(
rec['slot_CalibFlux_flux'])
photoCalMags[i] = testCal.instFluxToMagnitude(
rec['slot_CalibFlux_flux'], rec.getCentroid())
zptMeanCalMags[i] = fgcmZpt - 2.5 * np.log10(
rec['slot_CalibFlux_flux'])
# These should be very close but some tiny differences because the fgcm value
# is defined at the center of the bbox, and the photoCal is the mean over the box
self.assertFloatsAlmostEqual(photoCalMeanCalMags,
zptMeanCalMags,
rtol=1e-6)
# These should be roughly equal, but not precisely because of the focal-plane
# variation. However, this is a useful sanity check for something going totally
# wrong.
self.assertFloatsAlmostEqual(photoCalMeanCalMags,
photoCalMags,
rtol=1e-2)
# Test the transmission output
visitCatalog = butler.get('fgcmVisitCatalog')
lutCat = butler.get('fgcmLookUpTable')
testTrans = butler.get(
'transmission_atmosphere_fgcm',
dataId={visitDataRefName: visitCatalog[0]['visit']})
testResp = testTrans.sampleAt(position=afwGeom.Point2D(0, 0),
wavelengths=lutCat[0]['atmlambda'])
# The fit to be roughly consistent with the standard, although the
# airmass is taken into account even with the "frozen" atmosphere.
# This is also a rough comparison, because the interpolation does
# not work well with such a coarse look-up table used for the test.
self.assertFloatsAlmostEqual(testResp,
lutCat[0]['atmstdtrans'],
atol=0.06)
# The second should be close to the first, but there is the airmass
# difference so they aren't identical
testTrans2 = butler.get(
'transmission_atmosphere_fgcm',
dataId={visitDataRefName: visitCatalog[1]['visit']})
testResp2 = testTrans2.sampleAt(position=afwGeom.Point2D(0, 0),
wavelengths=lutCat[0]['atmlambda'])
self.assertFloatsAlmostEqual(testResp, testResp2, atol=1e-4)
def testIngest(self):
"""Test IngestIndexedReferenceTask."""
# Test with multiple files and standard config
config = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
IngestIndexedReferenceTask.parseAndRun(args=[
INPUT_DIR, "--output", self.outPath + "/output_multifile",
self.skyCatalogFile, self.skyCatalogFile
],
config=config)
# Test with config overrides
config2 = self.makeConfig(withRaDecErr=True,
withMagErr=True,
withPm=True,
withPmErr=True)
config2.ra_name = "ra"
config2.dec_name = "dec"
config2.dataset_config.ref_dataset_name = 'myrefcat'
# Change the indexing depth to prove we can.
# Smaller is better than larger because it makes fewer files.
config2.dataset_config.indexer.active.depth = self.depth - 1
config2.is_photometric_name = 'is_phot'
config2.is_resolved_name = 'is_res'
config2.is_variable_name = 'is_var'
config2.id_name = 'id'
config2.extra_col_names = ['val1', 'val2', 'val3']
config2.file_reader.header_lines = 1
config2.file_reader.colnames = [
'id',
'ra',
'dec',
'ra_err',
'dec_err',
'a',
'a_err',
'b',
'b_err',
'is_phot',
'is_res',
'is_var',
'val1',
'val2',
'val3',
'pm_ra',
'pm_dec',
'pm_ra_err',
'pm_dec_err',
'unixtime',
]
config2.file_reader.delimiter = '|'
# this also tests changing the delimiter
IngestIndexedReferenceTask.parseAndRun(args=[
INPUT_DIR, "--output", self.outPath + "/output_override",
self.skyCatalogFileDelim
],
config=config2)
# This location is known to have objects
cent = make_coord(93.0, -90.0)
# Test if we can get back the catalog with a non-standard dataset name
butler = dafPersist.Butler(self.outPath + "/output_override")
loaderConfig = LoadIndexedReferenceObjectsConfig()
loaderConfig.ref_dataset_name = "myrefcat"
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig)
cat = loader.loadSkyCircle(cent, self.searchRadius,
filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
# test that a catalog can be loaded even with a name not used for ingestion
butler = dafPersist.Butler(self.testRepoPath)
loaderConfig2 = LoadIndexedReferenceObjectsConfig()
loaderConfig2.ref_dataset_name = self.testDatasetName
loader = LoadIndexedReferenceObjectsTask(butler=butler,
config=loaderConfig2)
cat = loader.loadSkyCircle(cent, self.searchRadius,
filterName='a').refCat
self.assertTrue(len(cat) > 0)
self.assertTrue(cat.isContiguous())
def _testFgcmCalibrateTract(self, visits, tract, rawRepeatability,
filterNCalibMap):
"""
Test running of FgcmCalibrateTractTask
Parameters
----------
visits: `list`
List of visits to calibrate
tract: `int`
Tract number
rawRepeatability: `np.array`
Expected raw repeatability after convergence.
Length should be number of bands.
filterNCalibMap: `dict`
Mapping from filter name to number of photoCalibs created.
"""
args = [
self.inputDir, '--output', self.testDir, '--id',
'visit=' + '^'.join([str(visit) for visit in visits]),
'tract=%d' % (tract), '--doraise'
]
if len(self.configfiles) > 0:
args.extend(['--configfile', *self.configfiles])
args.extend(self.otherArgs)
# Move into the test directory so the plots will get cleaned in tearDown
# In the future, with Gen3, we will probably have a better way of managing
# non-data output such as plots.
cwd = os.getcwd()
os.chdir(self.testDir)
result = fgcmcal.FgcmCalibrateTractTableTask.parseAndRun(
args=args, config=self.config, doReturnResults=True)
self._checkResult(result)
# Move back to the previous directory
os.chdir(cwd)
# Check that the converged repeatability is what we expect
repeatability = result.resultList[0].results.repeatability
self.assertFloatsAlmostEqual(repeatability,
rawRepeatability,
atol=4e-6)
butler = dafPersist.butler.Butler(self.testDir)
# Check that the number of photoCalib objects in each filter are what we expect
for filterName in filterNCalibMap.keys():
subset = butler.subset('fgcm_tract_photoCalib',
tract=tract,
filter=filterName)
tot = 0
for dataRef in subset:
if butler.datasetExists('fgcm_tract_photoCalib',
dataId=dataRef.dataId):
tot += 1
self.assertEqual(tot, filterNCalibMap[filterName])
# Check that every visit got a transmission
visits = butler.queryMetadata('fgcm_tract_photoCalib', ('visit'),
tract=tract)
for visit in visits:
self.assertTrue(
butler.datasetExists('transmission_atmosphere_fgcm_tract',
tract=tract,
visit=visit))
# Check that we got the reference catalog output.
# This will raise an exception if the catalog is not there.
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = 'fgcm_stars_%d' % (tract)
task = LoadIndexedReferenceObjectsTask(butler, config=config)
coord = geom.SpherePoint(337.656174 * geom.degrees,
0.823595 * geom.degrees)
refStruct = task.loadSkyCircle(coord,
5.0 * geom.degrees,
filterName='r')
# Test the psf candidate counting, ratio should be between 0.0 and 1.0
candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) /
refStruct.refCat['r_nTotal'].astype(np.float64))
self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
# Test that temporary files aren't stored
self.assertFalse(butler.datasetExists('fgcmVisitCatalog'))
self.assertFalse(butler.datasetExists('fgcmStarObservations'))
self.assertFalse(butler.datasetExists('fgcmStarIndices'))
self.assertFalse(butler.datasetExists('fgcmReferenceStars'))
def _testFgcmOutputProducts(self, visitDataRefName, ccdDataRefName,
filterMapping, zpOffsets, testVisit, testCcd,
testFilter, testBandIndex):
"""
Test running of FgcmOutputProductsTask
Parameters
----------
visitDataRefName: `str`
Name of column in dataRef to get the visit
ccdDataRefName: `str`
Name of column in dataRef to get the ccd
filterMapping: `dict`
Mapping of filterName to dataRef filter names
zpOffsets: `np.array`
Zeropoint offsets expected
testVisit: `int`
Visit id to check for round-trip computations
testCcd: `int`
Ccd id to check for round-trip computations
testFilter: `str`
Filtername for testVisit/testCcd
testBandIndex: `int`
Band index for testVisit/testCcd
"""
args = [self.inputDir, '--output', self.testDir, '--doraise']
if len(self.configfiles) > 0:
args.extend(['--configfile', *self.configfiles])
args.extend(self.otherArgs)
result = fgcmcal.FgcmOutputProductsTask.parseAndRun(
args=args, config=self.config, doReturnResults=True)
self._checkResult(result)
# Extract the offsets from the results
offsets = result.resultList[0].results.offsets
self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6)
butler = dafPersist.butler.Butler(self.testDir)
# Test the reference catalog stars
# Read in the raw stars...
rawStars = butler.get('fgcmStandardStars',
fgcmcycle=self.config.cycleNumber)
# Read in the new reference catalog...
config = LoadIndexedReferenceObjectsConfig()
config.ref_dataset_name = 'fgcm_stars'
task = LoadIndexedReferenceObjectsTask(butler, config=config)
# Read in a giant radius to get them all
refStruct = task.loadSkyCircle(rawStars[0].getCoord(),
5.0 * geom.degrees,
filterName='r')
# Make sure all the stars are there
self.assertEqual(len(rawStars), len(refStruct.refCat))
# And make sure the numbers are consistent
test, = np.where(rawStars['id'][0] == refStruct.refCat['id'])
# Perform math on numpy arrays to maintain datatypes
mags = rawStars['mag_std_noabs'][:, 0].astype(np.float64) + offsets[0]
fluxes = (mags * units.ABmag).to_value(units.nJy)
fluxErrs = (np.log(10.) /
2.5) * fluxes * rawStars['magErr_std'][:, 0].astype(
np.float64)
# Only check the first one
self.assertFloatsAlmostEqual(fluxes[0],
refStruct.refCat['r_flux'][test[0]])
self.assertFloatsAlmostEqual(fluxErrs[0],
refStruct.refCat['r_fluxErr'][test[0]])
# Test the psf candidate counting, ratio should be between 0.0 and 1.0
candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) /
refStruct.refCat['r_nTotal'].astype(np.float64))
self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
# Test the fgcm_photoCalib output
zptCat = butler.get('fgcmZeropoints',
fgcmcycle=self.config.cycleNumber)
selected = (zptCat['fgcmFlag'] < 16)
# Read in all the calibrations, these should all be there
# This test is simply to ensure that all the photoCalib files exist
for rec in zptCat[selected]:
testCal = butler.get('fgcm_photoCalib',
dataId={
visitDataRefName: int(rec['visit']),
ccdDataRefName: int(rec['ccd']),
'filter': filterMapping[rec['filtername']]
})
self.assertIsNotNone(testCal)
# We do round-trip value checking on just the final one (chosen arbitrarily)
testCal = butler.get('fgcm_photoCalib',
dataId={
visitDataRefName: int(testVisit),
ccdDataRefName: int(testCcd),
'filter': filterMapping[testFilter]
})
self.assertIsNotNone(testCal)
src = butler.get('src',
dataId={
visitDataRefName: int(testVisit),
ccdDataRefName: int(testCcd)
})
# Only test sources with positive flux
gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0)
# We need to apply the calibration offset to the fgcmzpt (which is internal
# and doesn't know about that yet)
testZpInd, = np.where((zptCat['visit'] == testVisit)
& (zptCat['ccd'] == testCcd))
fgcmZpt = (zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex] +
zptCat['fgcmDeltaChrom'][testZpInd])
fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd])
if self.config.doComposeWcsJacobian:
# The raw zeropoint needs to be modified to know about the wcs jacobian
camera = butler.get('camera')
approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(
camera)
center = approxPixelAreaFields[testCcd].getBBox().getCenter()
pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center)
fgcmZpt += -2.5 * np.log10(pixAreaCorr)
# This is the magnitude through the mean calibration
photoCalMeanCalMags = np.zeros(gdSrc.sum())
# This is the magnitude through the full focal-plane variable mags
photoCalMags = np.zeros_like(photoCalMeanCalMags)
# This is the magnitude with the FGCM (central-ccd) zeropoint
zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
for i, rec in enumerate(src[gdSrc]):
photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(
rec['slot_CalibFlux_instFlux'])
photoCalMags[i] = testCal.instFluxToMagnitude(
rec['slot_CalibFlux_instFlux'], rec.getCentroid())
zptMeanCalMags[i] = fgcmZpt - 2.5 * np.log10(
rec['slot_CalibFlux_instFlux'])
# These should be very close but some tiny differences because the fgcm value
# is defined at the center of the bbox, and the photoCal is the mean over the box
self.assertFloatsAlmostEqual(photoCalMeanCalMags,
zptMeanCalMags,
rtol=1e-6)
# These should be roughly equal, but not precisely because of the focal-plane
# variation. However, this is a useful sanity check for something going totally
# wrong.
self.assertFloatsAlmostEqual(photoCalMeanCalMags,
photoCalMags,
rtol=1e-2)
# The next test compares the "FGCM standard magnitudes" (which are output
# from the fgcm code itself) to the "calibrated magnitudes" that are
# obtained from running photoCalib.calibrateCatalog() on the original
# src catalogs. This summary comparison ensures that using photoCalibs
# yields the same results as what FGCM is computing internally.
# Note that we additionally need to take into account the post-processing
# offsets used in the tests.
# For decent statistics, we are matching all the sources from one visit
# (multiple ccds)
subset = butler.subset('src',
dataId={visitDataRefName: int(testVisit)})
matchMag, matchDelta = self._getMatchedVisitCat(
rawStars, subset, testBandIndex, offsets)
st = np.argsort(matchMag)
# Compare the brightest 25% of stars. No matter the setting of
# deltaMagBkgOffsetPercentile, we want to ensure that these stars
# match on average.
brightest, = np.where(matchMag < matchMag[st[int(0.25 * st.size)]])
self.assertFloatsAlmostEqual(np.median(matchDelta[brightest]),
0.0,
atol=0.002)
# And the photoCal error is just the zeropoint gray error
self.assertFloatsAlmostEqual(
testCal.getCalibrationErr(), (np.log(10.0) / 2.5) *
testCal.getCalibrationMean() * fgcmZptGrayErr)
# Test the transmission output
visitCatalog = butler.get('fgcmVisitCatalog')
lutCat = butler.get('fgcmLookUpTable')
testTrans = butler.get(
'transmission_atmosphere_fgcm',
dataId={visitDataRefName: visitCatalog[0]['visit']})
testResp = testTrans.sampleAt(position=geom.Point2D(0, 0),
wavelengths=lutCat[0]['atmLambda'])
# The test fit is performed with the atmosphere parameters frozen
# (freezeStdAtmosphere = True). Thus the only difference between
# these output atmospheres and the standard is the different
# airmass. Furthermore, this is a very rough comparison because
# the look-up table is computed with very coarse sampling for faster
# testing.
# To account for overall throughput changes, we scale by the median ratio,
# we only care about the shape
ratio = np.median(testResp / lutCat[0]['atmStdTrans'])
self.assertFloatsAlmostEqual(testResp / ratio,
lutCat[0]['atmStdTrans'],
atol=0.04)
# The second should be close to the first, but there is the airmass
# difference so they aren't identical.
testTrans2 = butler.get(
'transmission_atmosphere_fgcm',
dataId={visitDataRefName: visitCatalog[1]['visit']})
testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0),
wavelengths=lutCat[0]['atmLambda'])
# As above, we scale by the ratio to compare the shape of the curve.
ratio = np.median(testResp / testResp2)
self.assertFloatsAlmostEqual(testResp / ratio, testResp2, atol=0.04)