def testIngest(self): """Test IngestIndexedReferenceTask.""" # Test with multiple files and standard config config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True) IngestIndexedReferenceTask.parseAndRun( args=[INPUT_DIR, "--output", self.outPath+"/output_multifile", self.skyCatalogFile, self.skyCatalogFile], config=config) # A newly-ingested refcat should be marked format_version=1. loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(self.outPath+"/output_multifile")) self.assertEqual(loader.dataset_config.format_version, 1) # Test with config overrides config2 = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True) config2.ra_name = "ra" config2.dec_name = "dec" config2.dataset_config.ref_dataset_name = 'myrefcat' # Change the indexing depth to prove we can. # Smaller is better than larger because it makes fewer files. config2.dataset_config.indexer.active.depth = self.depth - 1 config2.is_photometric_name = 'is_phot' config2.is_resolved_name = 'is_res' config2.is_variable_name = 'is_var' config2.id_name = 'id' config2.extra_col_names = ['val1', 'val2', 'val3'] config2.file_reader.header_lines = 1 config2.file_reader.colnames = [ 'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err', 'pm_dec_err', 'unixtime', ] config2.file_reader.delimiter = '|' # this also tests changing the delimiter IngestIndexedReferenceTask.parseAndRun( args=[INPUT_DIR, "--output", self.outPath+"/output_override", self.skyCatalogFileDelim], config=config2) # This location is known to have objects cent = make_coord(93.0, -90.0) # Test if we can get back the catalog with a non-standard dataset name butler = dafPersist.Butler(self.outPath+"/output_override") loaderConfig = LoadIndexedReferenceObjectsConfig() loaderConfig.ref_dataset_name = "myrefcat" loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig) cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat self.assertTrue(len(cat) > 0) self.assertTrue(cat.isContiguous()) # test that a catalog can be loaded even with a name not used for ingestion butler = dafPersist.Butler(self.testRepoPath) loaderConfig2 = LoadIndexedReferenceObjectsConfig() loaderConfig2.ref_dataset_name = self.testDatasetName loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2) cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat self.assertTrue(len(cat) > 0) self.assertTrue(cat.isContiguous())
def main(): import argparse class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): pass parser = argparse.ArgumentParser(description=__doc__, formatter_class=CustomFormatter) parser.add_argument( "refCatPath", help="Butler repo (written by IngestIndexedReferenceTask) containing the" " reference catalogs to test.") parser.add_argument( "inputGlob", help= "A glob pattern specifying the files (read by IngestIndexedReferenceTask) to " " check against the refCatPath output. (e.g. '/datasets/foo/*.csv'") parser.add_argument( "--nFiles", default=5, type=int, help="Number of input files to test (randomly selected from inputGlob)." ) parser.add_argument( "--nPerFile", default=100, type=int, help="Number of objects to test per file (randomly selected).") parser.add_argument( "--config", help= "A IngestIndexedReferenceConfig config file, for the field name mappings." ) parser.add_argument( "--ref_name", help="The name of the reference catalog stored in refCatPath.") args = parser.parse_args() ingestConfig = IngestIndexedReferenceConfig() ingestConfig.load(args.config) butler = lsst.daf.persistence.Butler(args.refCatPath) refObjConfig = LoadIndexedReferenceObjectsConfig() refObjConfig.ref_dataset_name = args.ref_name refObjLoader = LoadIndexedReferenceObjectsTask(butler, config=refObjConfig) reader = ingestConfig.file_reader.target() files = glob.glob(args.inputGlob) files.sort() for filename in random.sample(files, args.nFiles): do_one_file(filename, refObjLoader, reader, ingestConfig, args.nPerFile)
def runTest(withRaDecErr): # Generate a second catalog, with different ids inPath1 = tempfile.mkdtemp() skyCatalogFile1, _, skyCatalog1 = self.makeSkyCatalog(inPath1, idStart=25, seed=123) inPath2 = tempfile.mkdtemp() skyCatalogFile2, _, skyCatalog2 = self.makeSkyCatalog(inPath2, idStart=5432, seed=11) # override some field names, and use multiple cores config = ingestIndexTestBase.makeIngestIndexConfig(withRaDecErr=withRaDecErr, withMagErr=True, withPm=True, withPmErr=True) # use a very small HTM pixelization depth to ensure there will be collisions when # ingesting the files in parallel config.dataset_config.indexer.active.depth = 2 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that config.file_reader.format = 'ascii.commented_header' config.n_processes = 2 # use multiple cores for this test only config.id_name = 'id' # Use the ids from the generated catalogs outpath = os.path.join(self.outPath, "output_multifile_parallel", "_withRaDecErr" if withRaDecErr else "") IngestIndexedReferenceTask.parseAndRun( args=[self.input_dir, "--output", outpath, skyCatalogFile1, skyCatalogFile2], config=config) # Test if we can get back the catalog with a non-standard dataset name butler = dafPersist.Butler(outpath) loaderConfig = LoadIndexedReferenceObjectsConfig() loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig) self.checkAllRowsInRefcat(loader, skyCatalog1, config) self.checkAllRowsInRefcat(loader, skyCatalog2, config)
def testDefaultFilterAndFilterMap(self): """Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig.""" config = LoadIndexedReferenceObjectsConfig() config.defaultFilter = "b" config.filterMap = {"aprime": "a"} loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) for tupl, idList in self.compCats.items(): cent = make_coord(*tupl) lcat = loader.loadSkyCircle(cent, self.searchRadius) self.assertEqual(lcat.fluxField, "camFlux") if len(idList) > 0: defFluxFieldName = getRefFluxField(lcat.refCat.schema, None) self.assertTrue(defFluxFieldName in lcat.refCat.schema) aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime") self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema) break # just need one test
def testLoadIndexedReferenceConfig(self): """Make sure LoadIndexedReferenceConfig has needed fields.""" """ Including at least one from the base class LoadReferenceObjectsConfig """ config = LoadIndexedReferenceObjectsConfig() self.assertEqual(config.ref_dataset_name, "cal_ref_cat") self.assertEqual(config.defaultFilter, "")
def testIngestConfigOverrides(self): """Test IngestIndexedReferenceTask with different configs. """ config2 = makeIngestIndexConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True, withParallax=True) config2.ra_name = "ra" config2.dec_name = "dec" config2.dataset_config.ref_dataset_name = 'myrefcat' # Change the indexing depth to prove we can. # Smaller is better than larger because it makes fewer files. config2.dataset_config.indexer.active.depth = self.depth - 1 config2.is_photometric_name = 'is_phot' config2.is_resolved_name = 'is_res' config2.is_variable_name = 'is_var' config2.id_name = 'id' config2.extra_col_names = ['val1', 'val2', 'val3'] config2.file_reader.header_lines = 1 config2.file_reader.colnames = [ 'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err', 'pm_dec_err', 'parallax', 'parallax_err', 'unixtime', ] config2.file_reader.delimiter = '|' # this also tests changing the delimiter IngestIndexedReferenceTask.parseAndRun( args=[self.input_dir, "--output", self.outPath+"/output_override", self.skyCatalogFileDelim], config=config2) # Test if we can get back the catalog with a non-standard dataset name butler = dafPersist.Butler(self.outPath+"/output_override") loaderConfig = LoadIndexedReferenceObjectsConfig() loaderConfig.ref_dataset_name = "myrefcat" loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig) self.checkAllRowsInRefcat(loader, self.skyCatalog, config2) # test that a catalog can be loaded even with a name not used for ingestion butler = dafPersist.Butler(self.testRepoPath) loaderConfig2 = LoadIndexedReferenceObjectsConfig() loaderConfig2.ref_dataset_name = self.testDatasetName loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2) self.checkAllRowsInRefcat(loader, self.skyCatalog, config2)
def testIngest(self): """Test IngestIndexedReferenceTask.""" default_config = IngestIndexedReferenceTask.ConfigClass() # test ingest with default config # This should raise since I haven't specified the ra/dec/mag columns. with self.assertRaises(ValueError): IngestIndexedReferenceTask.parseAndRun(args=[ input_dir, "--output", self.out_path + "/output", self.sky_catalog_file ], config=default_config) # test with ~minimum config. Mag errors are not technically necessary, but might as well test here default_config.ra_name = 'ra_icrs' default_config.dec_name = 'dec_icrs' default_config.mag_column_list = ['a', 'b'] default_config.mag_err_column_map = {'a': 'a_err'} # should raise since all columns need an error column if any do with self.assertRaises(ValueError): IngestIndexedReferenceTask.parseAndRun(args=[ input_dir, "--output", self.out_path + "/output", self.sky_catalog_file ], config=default_config) # test with multiple files and correct config default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'} IngestIndexedReferenceTask.parseAndRun(args=[ input_dir, "--output", self.out_path + "/output_multifile", self.sky_catalog_file, self.sky_catalog_file ], config=default_config) # test with config overrides default_config = IngestIndexedReferenceTask.ConfigClass() default_config.ra_name = 'ra' default_config.dec_name = 'dec' default_config.mag_column_list = ['a', 'b'] default_config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'} default_config.dataset_config.ref_dataset_name = 'myrefcat' default_config.dataset_config.indexer.active.depth = 10 default_config.is_photometric_name = 'is_phot' default_config.is_resolved_name = 'is_res' default_config.is_variable_name = 'is_var' default_config.id_name = 'id' default_config.extra_col_names = ['val1', 'val2', 'val3'] default_config.file_reader.header_lines = 1 default_config.file_reader.colnames = [ 'id', 'ra', 'dec', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res', 'is_var', 'val1', 'val2', 'val3' ] default_config.file_reader.delimiter = '|' # this also tests changing the delimiter IngestIndexedReferenceTask.parseAndRun(args=[ input_dir, "--output", self.out_path + "/output_override", self.sky_catalog_file_delim ], config=default_config) # This location is known to have objects cent = make_coord(93.0, -90.0) # Test if we can get back the catalog with a non-standard dataset name butler = dafPersist.Butler(self.out_path + "/output_override") config = LoadIndexedReferenceObjectsConfig() config.ref_dataset_name = "myrefcat" loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config) cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a') self.assertTrue(len(cat) > 0) # test that a catalog can be loaded even with a name not used for ingestion butler = dafPersist.Butler(self.test_repo_path) config = LoadIndexedReferenceObjectsConfig() config.ref_dataset_name = self.test_dataset_name loader = LoadIndexedReferenceObjectsTask(butler=butler, config=config) cat = loader.loadSkyCircle(cent, self.search_radius, filterName='a') self.assertTrue(len(cat) > 0)
def testIngest(self): """Test IngestIndexedReferenceTask with different configs.""" # Test with multiple files and standard config config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True) # don't use the default depth, to avoid taking the time to create thousands of file locks config.dataset_config.indexer.active.depth = self.depth IngestIndexedReferenceTask.parseAndRun(args=[ self.input_dir, "--output", self.outPath + "/output_multifile", self.skyCatalogFile, self.skyCatalogFile ], config=config) # A newly-ingested refcat should be marked format_version=1. loader = LoadIndexedReferenceObjectsTask( butler=dafPersist.Butler(self.outPath + "/output_multifile")) self.assertEqual(loader.dataset_config.format_version, 1) # Test with config overrides config2 = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True) config2.ra_name = "ra" config2.dec_name = "dec" config2.dataset_config.ref_dataset_name = 'myrefcat' # Change the indexing depth to prove we can. # Smaller is better than larger because it makes fewer files. config2.dataset_config.indexer.active.depth = self.depth - 1 config2.is_photometric_name = 'is_phot' config2.is_resolved_name = 'is_res' config2.is_variable_name = 'is_var' config2.id_name = 'id' config2.extra_col_names = ['val1', 'val2', 'val3'] config2.file_reader.header_lines = 1 config2.file_reader.colnames = [ 'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err', 'pm_dec_err', 'unixtime', ] config2.file_reader.delimiter = '|' # this also tests changing the delimiter IngestIndexedReferenceTask.parseAndRun(args=[ self.input_dir, "--output", self.outPath + "/output_override", self.skyCatalogFileDelim ], config=config2) # Test if we can get back the catalog with a non-standard dataset name butler = dafPersist.Butler(self.outPath + "/output_override") loaderConfig = LoadIndexedReferenceObjectsConfig() loaderConfig.ref_dataset_name = "myrefcat" loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig) self.checkAllRowsInRefcat(loader, self.skyCatalog) # test that a catalog can be loaded even with a name not used for ingestion butler = dafPersist.Butler(self.testRepoPath) loaderConfig2 = LoadIndexedReferenceObjectsConfig() loaderConfig2.ref_dataset_name = self.testDatasetName loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2) self.checkAllRowsInRefcat(loader, self.skyCatalog)
def _runFgcmOutputProducts(self, visitDataRefName, ccdDataRefName, filterMapping, zpOffsets, testVisit, testCcd, testFilter, testBandIndex): """ """ if self.logLevel is not None: self.otherArgs.extend(['--loglevel', 'fgcmcal=%s' % self.logLevel]) args = [self.inputDir, '--output', self.testDir, '--doraise'] args.extend(self.otherArgs) result = fgcmcal.FgcmOutputProductsTask.parseAndRun( args=args, config=self.config, doReturnResults=True) self._checkResult(result) # Extract the offsets from the results offsets = result.resultList[0].results.offsets self.assertFloatsAlmostEqual(offsets[0], zpOffsets[0], rtol=1e-6) self.assertFloatsAlmostEqual(offsets[1], zpOffsets[1], rtol=1e-6) butler = dafPersistence.butler.Butler(self.testDir) # Test the reference catalog stars # Read in the raw stars... rawStars = butler.get('fgcmStandardStars', fgcmcycle=0) # Read in the new reference catalog... config = LoadIndexedReferenceObjectsConfig() config.ref_dataset_name = 'fgcm_stars' task = LoadIndexedReferenceObjectsTask(butler, config=config) # Read in a giant radius to get them all refStruct = task.loadSkyCircle(rawStars[0].getCoord(), 5.0 * lsst.geom.degrees, filterName='r') # Make sure all the stars are there self.assertEqual(len(rawStars), len(refStruct.refCat)) # And make sure the numbers are consistent test, = np.where(rawStars['id'][0] == refStruct.refCat['id']) mag = rawStars['mag_std_noabs'][0, 0] + offsets[0] flux = afwImage.fluxFromABMag(mag) fluxErr = afwImage.fluxErrFromABMagErr(rawStars['magerr_std'][0, 0], mag) self.assertFloatsAlmostEqual(flux, refStruct.refCat['r_flux'][test[0]], rtol=1e-6) self.assertFloatsAlmostEqual(fluxErr, refStruct.refCat['r_fluxErr'][test[0]], rtol=1e-6) # Test the joincal_photoCalib output zptCat = butler.get('fgcmZeropoints', fgcmcycle=0) selected = (zptCat['fgcmflag'] < 16) # Read in all the calibrations, these should all be there for rec in zptCat[selected]: testCal = butler.get('jointcal_photoCalib', dataId={ visitDataRefName: int(rec['visit']), ccdDataRefName: int(rec['ccd']), 'filter': filterMapping[rec['filtername']], 'tract': 0 }) # Our round-trip tests will be on this final one which is still loaded testCal = butler.get('jointcal_photoCalib', dataId={ visitDataRefName: int(testVisit), ccdDataRefName: int(testCcd), 'filter': filterMapping[testFilter], 'tract': 0 }) src = butler.get('src', dataId={ visitDataRefName: int(testVisit), ccdDataRefName: int(testCcd) }) # Only test sources with positive flux gdSrc = (src['slot_CalibFlux_flux'] > 0.0) # We need to apply the calibration offset to the fgcmzpt (which is internal # and doesn't know about that yet) testZpInd, = np.where((zptCat['visit'] == testVisit) & (zptCat['ccd'] == testCcd)) fgcmZpt = zptCat['fgcmzpt'][testZpInd] + offsets[testBandIndex] # This is the magnitude through the mean calibration photoCalMeanCalMags = np.zeros(gdSrc.sum()) # This is the magnitude through the full focal-plane variable mags photoCalMags = np.zeros_like(photoCalMeanCalMags) # This is the magnitude with the FGCM (central-ccd) zeropoint zptMeanCalMags = np.zeros_like(photoCalMeanCalMags) for i, rec in enumerate(src[gdSrc]): photoCalMeanCalMags[i] = testCal.instFluxToMagnitude( rec['slot_CalibFlux_flux']) photoCalMags[i] = testCal.instFluxToMagnitude( rec['slot_CalibFlux_flux'], rec.getCentroid()) zptMeanCalMags[i] = fgcmZpt - 2.5 * np.log10( rec['slot_CalibFlux_flux']) # These should be very close but some tiny differences because the fgcm value # is defined at the center of the bbox, and the photoCal is the mean over the box self.assertFloatsAlmostEqual(photoCalMeanCalMags, zptMeanCalMags, rtol=1e-6) # These should be roughly equal, but not precisely because of the focal-plane # variation. However, this is a useful sanity check for something going totally # wrong. self.assertFloatsAlmostEqual(photoCalMeanCalMags, photoCalMags, rtol=1e-2) # Test the transmission output visitCatalog = butler.get('fgcmVisitCatalog') lutCat = butler.get('fgcmLookUpTable') testTrans = butler.get( 'transmission_atmosphere_fgcm', dataId={visitDataRefName: visitCatalog[0]['visit']}) testResp = testTrans.sampleAt(position=afwGeom.Point2D(0, 0), wavelengths=lutCat[0]['atmlambda']) # The fit to be roughly consistent with the standard, although the # airmass is taken into account even with the "frozen" atmosphere. # This is also a rough comparison, because the interpolation does # not work well with such a coarse look-up table used for the test. self.assertFloatsAlmostEqual(testResp, lutCat[0]['atmstdtrans'], atol=0.06) # The second should be close to the first, but there is the airmass # difference so they aren't identical testTrans2 = butler.get( 'transmission_atmosphere_fgcm', dataId={visitDataRefName: visitCatalog[1]['visit']}) testResp2 = testTrans2.sampleAt(position=afwGeom.Point2D(0, 0), wavelengths=lutCat[0]['atmlambda']) self.assertFloatsAlmostEqual(testResp, testResp2, atol=1e-4)
def testIngest(self): """Test IngestIndexedReferenceTask.""" # Test with multiple files and standard config config = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True) IngestIndexedReferenceTask.parseAndRun(args=[ INPUT_DIR, "--output", self.outPath + "/output_multifile", self.skyCatalogFile, self.skyCatalogFile ], config=config) # Test with config overrides config2 = self.makeConfig(withRaDecErr=True, withMagErr=True, withPm=True, withPmErr=True) config2.ra_name = "ra" config2.dec_name = "dec" config2.dataset_config.ref_dataset_name = 'myrefcat' # Change the indexing depth to prove we can. # Smaller is better than larger because it makes fewer files. config2.dataset_config.indexer.active.depth = self.depth - 1 config2.is_photometric_name = 'is_phot' config2.is_resolved_name = 'is_res' config2.is_variable_name = 'is_var' config2.id_name = 'id' config2.extra_col_names = ['val1', 'val2', 'val3'] config2.file_reader.header_lines = 1 config2.file_reader.colnames = [ 'id', 'ra', 'dec', 'ra_err', 'dec_err', 'a', 'a_err', 'b', 'b_err', 'is_phot', 'is_res', 'is_var', 'val1', 'val2', 'val3', 'pm_ra', 'pm_dec', 'pm_ra_err', 'pm_dec_err', 'unixtime', ] config2.file_reader.delimiter = '|' # this also tests changing the delimiter IngestIndexedReferenceTask.parseAndRun(args=[ INPUT_DIR, "--output", self.outPath + "/output_override", self.skyCatalogFileDelim ], config=config2) # This location is known to have objects cent = make_coord(93.0, -90.0) # Test if we can get back the catalog with a non-standard dataset name butler = dafPersist.Butler(self.outPath + "/output_override") loaderConfig = LoadIndexedReferenceObjectsConfig() loaderConfig.ref_dataset_name = "myrefcat" loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig) cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat self.assertTrue(len(cat) > 0) self.assertTrue(cat.isContiguous()) # test that a catalog can be loaded even with a name not used for ingestion butler = dafPersist.Butler(self.testRepoPath) loaderConfig2 = LoadIndexedReferenceObjectsConfig() loaderConfig2.ref_dataset_name = self.testDatasetName loader = LoadIndexedReferenceObjectsTask(butler=butler, config=loaderConfig2) cat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a').refCat self.assertTrue(len(cat) > 0) self.assertTrue(cat.isContiguous())
def _testFgcmCalibrateTract(self, visits, tract, rawRepeatability, filterNCalibMap): """ Test running of FgcmCalibrateTractTask Parameters ---------- visits: `list` List of visits to calibrate tract: `int` Tract number rawRepeatability: `np.array` Expected raw repeatability after convergence. Length should be number of bands. filterNCalibMap: `dict` Mapping from filter name to number of photoCalibs created. """ args = [ self.inputDir, '--output', self.testDir, '--id', 'visit=' + '^'.join([str(visit) for visit in visits]), 'tract=%d' % (tract), '--doraise' ] if len(self.configfiles) > 0: args.extend(['--configfile', *self.configfiles]) args.extend(self.otherArgs) # Move into the test directory so the plots will get cleaned in tearDown # In the future, with Gen3, we will probably have a better way of managing # non-data output such as plots. cwd = os.getcwd() os.chdir(self.testDir) result = fgcmcal.FgcmCalibrateTractTableTask.parseAndRun( args=args, config=self.config, doReturnResults=True) self._checkResult(result) # Move back to the previous directory os.chdir(cwd) # Check that the converged repeatability is what we expect repeatability = result.resultList[0].results.repeatability self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=4e-6) butler = dafPersist.butler.Butler(self.testDir) # Check that the number of photoCalib objects in each filter are what we expect for filterName in filterNCalibMap.keys(): subset = butler.subset('fgcm_tract_photoCalib', tract=tract, filter=filterName) tot = 0 for dataRef in subset: if butler.datasetExists('fgcm_tract_photoCalib', dataId=dataRef.dataId): tot += 1 self.assertEqual(tot, filterNCalibMap[filterName]) # Check that every visit got a transmission visits = butler.queryMetadata('fgcm_tract_photoCalib', ('visit'), tract=tract) for visit in visits: self.assertTrue( butler.datasetExists('transmission_atmosphere_fgcm_tract', tract=tract, visit=visit)) # Check that we got the reference catalog output. # This will raise an exception if the catalog is not there. config = LoadIndexedReferenceObjectsConfig() config.ref_dataset_name = 'fgcm_stars_%d' % (tract) task = LoadIndexedReferenceObjectsTask(butler, config=config) coord = geom.SpherePoint(337.656174 * geom.degrees, 0.823595 * geom.degrees) refStruct = task.loadSkyCircle(coord, 5.0 * geom.degrees, filterName='r') # Test the psf candidate counting, ratio should be between 0.0 and 1.0 candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) / refStruct.refCat['r_nTotal'].astype(np.float64)) self.assertFloatsAlmostEqual(candRatio.min(), 0.0) self.assertFloatsAlmostEqual(candRatio.max(), 1.0) # Test that temporary files aren't stored self.assertFalse(butler.datasetExists('fgcmVisitCatalog')) self.assertFalse(butler.datasetExists('fgcmStarObservations')) self.assertFalse(butler.datasetExists('fgcmStarIndices')) self.assertFalse(butler.datasetExists('fgcmReferenceStars'))
def _testFgcmOutputProducts(self, visitDataRefName, ccdDataRefName, filterMapping, zpOffsets, testVisit, testCcd, testFilter, testBandIndex): """ Test running of FgcmOutputProductsTask Parameters ---------- visitDataRefName: `str` Name of column in dataRef to get the visit ccdDataRefName: `str` Name of column in dataRef to get the ccd filterMapping: `dict` Mapping of filterName to dataRef filter names zpOffsets: `np.array` Zeropoint offsets expected testVisit: `int` Visit id to check for round-trip computations testCcd: `int` Ccd id to check for round-trip computations testFilter: `str` Filtername for testVisit/testCcd testBandIndex: `int` Band index for testVisit/testCcd """ args = [self.inputDir, '--output', self.testDir, '--doraise'] if len(self.configfiles) > 0: args.extend(['--configfile', *self.configfiles]) args.extend(self.otherArgs) result = fgcmcal.FgcmOutputProductsTask.parseAndRun( args=args, config=self.config, doReturnResults=True) self._checkResult(result) # Extract the offsets from the results offsets = result.resultList[0].results.offsets self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6) butler = dafPersist.butler.Butler(self.testDir) # Test the reference catalog stars # Read in the raw stars... rawStars = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber) # Read in the new reference catalog... config = LoadIndexedReferenceObjectsConfig() config.ref_dataset_name = 'fgcm_stars' task = LoadIndexedReferenceObjectsTask(butler, config=config) # Read in a giant radius to get them all refStruct = task.loadSkyCircle(rawStars[0].getCoord(), 5.0 * geom.degrees, filterName='r') # Make sure all the stars are there self.assertEqual(len(rawStars), len(refStruct.refCat)) # And make sure the numbers are consistent test, = np.where(rawStars['id'][0] == refStruct.refCat['id']) # Perform math on numpy arrays to maintain datatypes mags = rawStars['mag_std_noabs'][:, 0].astype(np.float64) + offsets[0] fluxes = (mags * units.ABmag).to_value(units.nJy) fluxErrs = (np.log(10.) / 2.5) * fluxes * rawStars['magErr_std'][:, 0].astype( np.float64) # Only check the first one self.assertFloatsAlmostEqual(fluxes[0], refStruct.refCat['r_flux'][test[0]]) self.assertFloatsAlmostEqual(fluxErrs[0], refStruct.refCat['r_fluxErr'][test[0]]) # Test the psf candidate counting, ratio should be between 0.0 and 1.0 candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) / refStruct.refCat['r_nTotal'].astype(np.float64)) self.assertFloatsAlmostEqual(candRatio.min(), 0.0) self.assertFloatsAlmostEqual(candRatio.max(), 1.0) # Test the fgcm_photoCalib output zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber) selected = (zptCat['fgcmFlag'] < 16) # Read in all the calibrations, these should all be there # This test is simply to ensure that all the photoCalib files exist for rec in zptCat[selected]: testCal = butler.get('fgcm_photoCalib', dataId={ visitDataRefName: int(rec['visit']), ccdDataRefName: int(rec['ccd']), 'filter': filterMapping[rec['filtername']] }) self.assertIsNotNone(testCal) # We do round-trip value checking on just the final one (chosen arbitrarily) testCal = butler.get('fgcm_photoCalib', dataId={ visitDataRefName: int(testVisit), ccdDataRefName: int(testCcd), 'filter': filterMapping[testFilter] }) self.assertIsNotNone(testCal) src = butler.get('src', dataId={ visitDataRefName: int(testVisit), ccdDataRefName: int(testCcd) }) # Only test sources with positive flux gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0) # We need to apply the calibration offset to the fgcmzpt (which is internal # and doesn't know about that yet) testZpInd, = np.where((zptCat['visit'] == testVisit) & (zptCat['ccd'] == testCcd)) fgcmZpt = (zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex] + zptCat['fgcmDeltaChrom'][testZpInd]) fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd]) if self.config.doComposeWcsJacobian: # The raw zeropoint needs to be modified to know about the wcs jacobian camera = butler.get('camera') approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields( camera) center = approxPixelAreaFields[testCcd].getBBox().getCenter() pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center) fgcmZpt += -2.5 * np.log10(pixAreaCorr) # This is the magnitude through the mean calibration photoCalMeanCalMags = np.zeros(gdSrc.sum()) # This is the magnitude through the full focal-plane variable mags photoCalMags = np.zeros_like(photoCalMeanCalMags) # This is the magnitude with the FGCM (central-ccd) zeropoint zptMeanCalMags = np.zeros_like(photoCalMeanCalMags) for i, rec in enumerate(src[gdSrc]): photoCalMeanCalMags[i] = testCal.instFluxToMagnitude( rec['slot_CalibFlux_instFlux']) photoCalMags[i] = testCal.instFluxToMagnitude( rec['slot_CalibFlux_instFlux'], rec.getCentroid()) zptMeanCalMags[i] = fgcmZpt - 2.5 * np.log10( rec['slot_CalibFlux_instFlux']) # These should be very close but some tiny differences because the fgcm value # is defined at the center of the bbox, and the photoCal is the mean over the box self.assertFloatsAlmostEqual(photoCalMeanCalMags, zptMeanCalMags, rtol=1e-6) # These should be roughly equal, but not precisely because of the focal-plane # variation. However, this is a useful sanity check for something going totally # wrong. self.assertFloatsAlmostEqual(photoCalMeanCalMags, photoCalMags, rtol=1e-2) # The next test compares the "FGCM standard magnitudes" (which are output # from the fgcm code itself) to the "calibrated magnitudes" that are # obtained from running photoCalib.calibrateCatalog() on the original # src catalogs. This summary comparison ensures that using photoCalibs # yields the same results as what FGCM is computing internally. # Note that we additionally need to take into account the post-processing # offsets used in the tests. # For decent statistics, we are matching all the sources from one visit # (multiple ccds) subset = butler.subset('src', dataId={visitDataRefName: int(testVisit)}) matchMag, matchDelta = self._getMatchedVisitCat( rawStars, subset, testBandIndex, offsets) st = np.argsort(matchMag) # Compare the brightest 25% of stars. No matter the setting of # deltaMagBkgOffsetPercentile, we want to ensure that these stars # match on average. brightest, = np.where(matchMag < matchMag[st[int(0.25 * st.size)]]) self.assertFloatsAlmostEqual(np.median(matchDelta[brightest]), 0.0, atol=0.002) # And the photoCal error is just the zeropoint gray error self.assertFloatsAlmostEqual( testCal.getCalibrationErr(), (np.log(10.0) / 2.5) * testCal.getCalibrationMean() * fgcmZptGrayErr) # Test the transmission output visitCatalog = butler.get('fgcmVisitCatalog') lutCat = butler.get('fgcmLookUpTable') testTrans = butler.get( 'transmission_atmosphere_fgcm', dataId={visitDataRefName: visitCatalog[0]['visit']}) testResp = testTrans.sampleAt(position=geom.Point2D(0, 0), wavelengths=lutCat[0]['atmLambda']) # The test fit is performed with the atmosphere parameters frozen # (freezeStdAtmosphere = True). Thus the only difference between # these output atmospheres and the standard is the different # airmass. Furthermore, this is a very rough comparison because # the look-up table is computed with very coarse sampling for faster # testing. # To account for overall throughput changes, we scale by the median ratio, # we only care about the shape ratio = np.median(testResp / lutCat[0]['atmStdTrans']) self.assertFloatsAlmostEqual(testResp / ratio, lutCat[0]['atmStdTrans'], atol=0.04) # The second should be close to the first, but there is the airmass # difference so they aren't identical. testTrans2 = butler.get( 'transmission_atmosphere_fgcm', dataId={visitDataRefName: visitCatalog[1]['visit']}) testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0), wavelengths=lutCat[0]['atmLambda']) # As above, we scale by the ratio to compare the shape of the curve. ratio = np.median(testResp / testResp2) self.assertFloatsAlmostEqual(testResp / ratio, testResp2, atol=0.04)