def testCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. """ testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests','scratchSpace','phoSimTestCatalog.txt') testBulge.write_catalog(catName) testDisk.write_catalog(catName, write_header=False, write_mode='a') testAgn.write_catalog(catName, write_header=False, write_mode='a') testStar.write_catalog(catName, write_header=False, write_mode='a') testFile = open(catName,'r') testLines = testFile.readlines() testFile.close() controlLines = self.baseLineFile.readlines() for line in testLines: msg = '%s not in controlLines' % line self.assertTrue(line in controlLines, msg=msg) for line in controlLines: msg = '%s not in testLines' self.assertTrue(line in testLines, msg=msg) if os.path.exists(catName): os.unlink(catName)
def testCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. """ testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata=self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata=self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata=self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata=self.obs_metadata) catName = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'phoSimTestCatalog.txt') testBulge.phoSimHeaderMap = test_header_map testBulge.write_catalog(catName) testDisk.write_catalog(catName, write_header=False, write_mode='a') testAgn.write_catalog(catName, write_header=False, write_mode='a') testStar.write_catalog(catName, write_header=False, write_mode='a') self.verify_catalog(catName) if os.path.exists(catName): os.unlink(catName)
def testZPointSourceSchema(self): """ Create a PhoSim InstanceCatalog of extra-galactic point sources (agns). Verify that the schema of the actual objects conforms to what PhoSim expects, as defined here https://bitbucket.org/phosim/phosim_release/wiki/Instance%20Catalog """ cat = PhoSimCatalogZPoint(self.agnDB, obs_metadata=self.obs_metadata) cat.phoSimHeaderMap = test_header_map cat_name = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'phosim_agn_schema_cat.txt') if os.path.exists(cat_name): os.unlink(cat_name) cat.write_catalog(cat_name) with open(cat_name, 'r') as input_file: cat_lines = input_file.readlines() n_obj = 0 for line in cat_lines: params = line.split() if len(params) > 2: n_obj += 1 self.assertEqual(len(params), 17) self.assertEqual(params[0], 'object') self.assertEqual(round(float(params[1])), float(params[1]), 10) # id float(params[2]) # ra float(params[3]) # dec float(params[4]) # mag norm self.assertIn('agnSED', params[5]) # sed name self.assertGreater(float(params[6]), 0.0) # redshift self.assertAlmostEqual(float(params[7]), 0.0, 10) # gamma1 self.assertAlmostEqual(float(params[8]), 0.0, 10) # gamma2 self.assertAlmostEqual(float(params[9]), 0.0, 10) # kappa self.assertAlmostEqual(float(params[10]), 0.0, 10) # delta_ra self.assertAlmostEqual(float(params[11]), 0.0, 10) # delta_dec self.assertEqual(params[12], 'point') # source type dust_msg = ( 'It is possible you are outputting Milky Way dust parameters before ' 'internal dust parameters; internal dust should come first' ) self.assertEqual(params[13], 'none', msg=dust_msg) # internal dust self.assertEqual(params[14], 'CCM', msg=dust_msg) # Milky Way dust self.assertGreater(float(params[15]), 0.0, msg=dust_msg) # Av self.assertAlmostEqual(float(params[16]), 3.1, msg=dust_msg) # Rv self.assertGreater(n_obj, 0) if os.path.exists(cat_name): os.unlink(cat_name)
def testCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. """ testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) testBulge.phoSimHeaderMap = test_header_map with lsst.utils.tests.getTempFilePath('.txt') as catName: testBulge.write_catalog(catName) testDisk.write_catalog(catName, write_header=False, write_mode='a') testAgn.write_catalog(catName, write_header=False, write_mode='a') testStar.write_catalog(catName, write_header=False, write_mode='a') self.verify_catalog(catName)
def testZPointSourceSchema(self): """ Create a PhoSim InstanceCatalog of extra-galactic point sources (agns). Verify that the schema of the actual objects conforms to what PhoSim expects, as defined here https://bitbucket.org/phosim/phosim_release/wiki/Instance%20Catalog """ cat = PhoSimCatalogZPoint(self.agnDB, obs_metadata=self.obs_metadata) cat.phoSimHeaderMap = test_header_map with lsst.utils.tests.getTempFilePath('.txt') as cat_name: cat.write_catalog(cat_name) with open(cat_name, 'r') as input_file: cat_lines = input_file.readlines() n_obj = 0 for line in cat_lines: params = line.split() if len(params) > 2: n_obj += 1 self.assertEqual(len(params), 17) self.assertEqual(params[0], 'object') self.assertEqual(round(float(params[1])), float(params[1]), 10) # id float(params[2]) # ra float(params[3]) # dec float(params[4]) # mag norm self.assertIn('agnSED', params[5]) # sed name self.assertGreater(float(params[6]), 0.0) # redshift self.assertAlmostEqual(float(params[7]), 0.0, 10) # gamma1 self.assertAlmostEqual(float(params[8]), 0.0, 10) # gamma2 self.assertAlmostEqual(float(params[9]), 0.0, 10) # kappa self.assertAlmostEqual(float(params[10]), 0.0, 10) # delta_ra self.assertAlmostEqual(float(params[11]), 0.0, 10) # delta_dec self.assertEqual(params[12], 'point') # source type dust_msg = ('It is possible you are outputting Milky Way dust parameters before ' 'internal dust parameters; internal dust should come first') self.assertEqual(params[13], 'none', msg=dust_msg) # internal dust self.assertEqual(params[14], 'CCM', msg=dust_msg) # Milky Way dust self.assertGreater(float(params[15]), 0.0, msg=dust_msg) # Av self.assertAlmostEqual(float(params[16]), 3.1, msg=dust_msg) # Rv self.assertGreater(n_obj, 0)
def testCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. """ testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata=self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata=self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata=self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata=self.obs_metadata) testBulge.phoSimHeaderMap = test_header_map with lsst.utils.tests.getTempFilePath('.txt') as catName: testBulge.write_catalog(catName) testDisk.write_catalog(catName, write_header=False, write_mode='a') testAgn.write_catalog(catName, write_header=False, write_mode='a') testStar.write_catalog(catName, write_header=False, write_mode='a') self.verify_catalog(catName)
def testCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. """ testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests','scratchSpace','phoSimTestCatalog.txt') testBulge.write_catalog(catName) testDisk.write_catalog(catName, write_header=False, write_mode='a') testAgn.write_catalog(catName, write_header=False, write_mode='a') testStar.write_catalog(catName, write_header=False, write_mode='a') self.verify_catalog(catName) if os.path.exists(catName): os.unlink(catName)
def testGalSimPhoSimCat(self): """ Run a GalSimPhoSim catalog on some data. Then, generate an ordinary PhoSim catalog using the same data. Verify that the two resulting PhoSim catalogs are identical. """ galsim_cat_name = os.path.join(self.dataDir, 'galSimPhoSim_galsim_cat.txt') phosim_cat_name = os.path.join(self.dataDir, 'galSimPhoSim_phosim_cat.txt') galsim_image_root = os.path.join(self.dataDir, 'galSimPhoSim_images') db = fileDBObject(self.bulge_name, dtype=self.dtype, runtable='test_bulges', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 55 gs_cat = GalSimPhoSimGalaxies(db, obs_metadata=self.obs) gs_cat.camera_wrapper = GalSimCameraWrapper(self.camera) gs_cat.bandpassNames = self.obs.bandpass gs_cat.PSF = SNRdocumentPSF() gs_cat.phoSimHeaderMap = {} gs_cat.write_catalog(galsim_cat_name) gs_cat_0 = gs_cat ps_cat = PhoSimCatalogSersic2D(db, obs_metadata=self.obs) ps_cat.phoSimHeaderMap = {} ps_cat.write_catalog(phosim_cat_name) db = fileDBObject(self.disk_name, dtype=self.dtype, runtable='test_disks', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 155 gs_cat = GalSimPhoSimGalaxies(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.copyGalSimInterpreter(gs_cat_0) gs_cat.write_catalog(galsim_cat_name, write_header=False, write_mode='a') gs_cat_0 = gs_cat ps_cat = PhoSimCatalogSersic2D(db, obs_metadata=self.obs) ps_cat.write_catalog(phosim_cat_name, write_header=False, write_mode='a') db = fileDBObject(self.agn_name, dtype=self.dtype, runtable='test_agn', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 255 gs_cat = GalSimPhoSimAgn(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.copyGalSimInterpreter(gs_cat_0) gs_cat.write_catalog(galsim_cat_name, write_header=False, write_mode='a') gs_cat_0 = gs_cat ps_cat = PhoSimCatalogZPoint(db, obs_metadata=self.obs) ps_cat.write_catalog(phosim_cat_name, write_header=False, write_mode='a') db = fileDBObject(self.star_name, dtype=self.dtype, runtable='test_agn', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 255 gs_cat = GalSimPhoSimStars(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.copyGalSimInterpreter(gs_cat_0) gs_cat.write_catalog(galsim_cat_name, write_header=False, write_mode='a') ps_cat = PhoSimCatalogPoint(db, obs_metadata=self.obs) ps_cat.write_catalog(phosim_cat_name, write_header=False, write_mode='a') written_files = gs_cat.write_images(nameRoot=galsim_image_root) self.assertGreater(len(written_files), 0) for name in written_files: os.unlink(name) with open(galsim_cat_name, 'r') as galsim_input: with open(phosim_cat_name, 'r') as phosim_input: galsim_lines = galsim_input.readlines() phosim_lines = phosim_input.readlines() self.assertEqual(len(galsim_lines), len(phosim_lines)) self.assertEqual(len(galsim_lines), 4 * self.n_objects + 7) for line in galsim_lines: self.assertIn(line, phosim_lines) for line in phosim_lines: self.assertIn(line, galsim_lines) if os.path.exists(galsim_cat_name): os.unlink(galsim_cat_name) if os.path.exists(phosim_cat_name): os.unlink(phosim_cat_name)
def testCompoundCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. This test uses CompoundInstanceCatalog """ # first, generate the catalog without a CompoundInstanceCatalog single_catName = tempfile.mktemp(dir=ROOT, prefix='phoSimTestCatalog_single', suffix='.txt') testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata=self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata=self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata=self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata=self.obs_metadata) testBulge.phoSimHeaderMap = test_header_map testBulge.write_catalog(single_catName) testDisk.write_catalog(single_catName, write_header=False, write_mode='a') testAgn.write_catalog(single_catName, write_header=False, write_mode='a') testStar.write_catalog(single_catName, write_header=False, write_mode='a') # now, generate the catalog using CompoundInstanceCatalog # # because the CompoundCatalogDBObject requires that database # connection parameters be set in the input CatalogDBObject # daughter class definitions, we have to declare dummy # CatalogDBObject daughter classes below class dummyDBbase(object): driver = 'sqlite' database = 'PhoSimTestDatabase.db' dummyDBbase.database = self.tempDB class dummyBulgeDB(dummyDBbase, testGalaxyBulgeDBObj): objid = 'dummy_bulge' class dummyDiskDB(dummyDBbase, testGalaxyDiskDBObj): objid = 'dummy_disk' class dummyAgnDB(dummyDBbase, testGalaxyAgnDBObj): objid = 'dummy_agn' class dummyStarDB(dummyDBbase, testStarsDBObj): objid = 'dummy_stars' compoundCatalog = CompoundInstanceCatalog( [ PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, PhoSimCatalogZPoint, PhoSimCatalogPoint ], [dummyBulgeDB, dummyDiskDB, dummyAgnDB, dummyStarDB], obs_metadata=self.obs_metadata) self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3) compound_catName = tempfile.mktemp(dir=ROOT, prefix='phoSimTestCatalog_compound', suffix='.txt') compoundCatalog.phoSimHeaderMap = test_header_map compoundCatalog.write_catalog(compound_catName) # verify that the compound catalog is what we expect self.verify_catalog(compound_catName) # verify that the two catalogs are equivalent with open(single_catName, 'r') as single_file: with open(compound_catName, 'r') as compound_file: single_lines = single_file.readlines() compound_lines = compound_file.readlines() for line in single_lines: self.assertIn(line, compound_lines) for line in compound_lines: self.assertIn(line, single_lines) if os.path.exists(compound_catName): os.unlink(compound_catName) if os.path.exists(single_catName): os.unlink(single_catName)
def testGalSimPhoSimCat(self): """ Run a GalSimPhoSim catalog on some data. Then, generate an ordinary PhoSim catalog using the same data. Verify that the two resulting PhoSim catalogs are identical. """ galsim_cat_name = os.path.join(self.dataDir, 'galSimPhoSim_galsim_cat.txt') phosim_cat_name = os.path.join(self.dataDir, 'galSimPhoSim_phosim_cat.txt') galsim_image_root = os.path.join(self.dataDir, 'galSimPhoSim_images') db = fileDBObject(self.bulge_name, dtype=self.dtype, runtable='test_bulges', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 55 gs_cat = GalSimPhoSimGalaxies(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.PSF = SNRdocumentPSF() gs_cat.phoSimHeaderMap = {} gs_cat.write_catalog(galsim_cat_name) gs_cat_0 = gs_cat ps_cat = PhoSimCatalogSersic2D(db, obs_metadata=self.obs) ps_cat.phoSimHeaderMap = {} ps_cat.write_catalog(phosim_cat_name) db = fileDBObject(self.disk_name, dtype=self.dtype, runtable='test_disks', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 155 gs_cat = GalSimPhoSimGalaxies(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.copyGalSimInterpreter(gs_cat_0) gs_cat.write_catalog(galsim_cat_name, write_header=False, write_mode='a') gs_cat_0 = gs_cat ps_cat = PhoSimCatalogSersic2D(db, obs_metadata=self.obs) ps_cat.write_catalog(phosim_cat_name, write_header=False, write_mode='a') db = fileDBObject(self.agn_name, dtype=self.dtype, runtable='test_agn', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 255 gs_cat = GalSimPhoSimAgn(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.copyGalSimInterpreter(gs_cat_0) gs_cat.write_catalog(galsim_cat_name, write_header=False, write_mode='a') gs_cat_0 = gs_cat ps_cat = PhoSimCatalogZPoint(db, obs_metadata=self.obs) ps_cat.write_catalog(phosim_cat_name, write_header=False, write_mode='a') db = fileDBObject(self.star_name, dtype=self.dtype, runtable='test_agn', idColKey='id') db.raColName = 'ra_deg' db.decColName = 'dec_deg' db.objectTypeId = 255 gs_cat = GalSimPhoSimStars(db, obs_metadata=self.obs) gs_cat.bandpassNames = self.obs.bandpass gs_cat.copyGalSimInterpreter(gs_cat_0) gs_cat.write_catalog(galsim_cat_name, write_header=False, write_mode='a') ps_cat = PhoSimCatalogPoint(db, obs_metadata=self.obs) ps_cat.write_catalog(phosim_cat_name, write_header=False, write_mode='a') written_files = gs_cat.write_images(nameRoot=galsim_image_root) self.assertGreater(len(written_files), 0) for name in written_files: os.unlink(name) with open(galsim_cat_name, 'r') as galsim_input: with open(phosim_cat_name, 'r') as phosim_input: galsim_lines = galsim_input.readlines() phosim_lines = phosim_input.readlines() self.assertEqual(len(galsim_lines), len(phosim_lines)) self.assertEqual(len(galsim_lines), 4*self.n_objects+7) for line in galsim_lines: self.assertIn(line, phosim_lines) for line in phosim_lines: self.assertIn(line, galsim_lines) if os.path.exists(galsim_cat_name): os.unlink(galsim_cat_name) if os.path.exists(phosim_cat_name): os.unlink(phosim_cat_name)
bright_cat.min_mag = args.min_mag from lsst.sims.catalogs.definitions import parallelCatalogWriter cat_dict = {} cat_dict[os.path.join(out_dir, star_name)] = star_cat cat_dict[os.path.join(out_dir, 'bright_stars_%d.txt' % obshistid)] = bright_cat parallelCatalogWriter(cat_dict, chunk_size=100000, write_header=False) cat = PhoSimCatalogSersic2D(bulge_db, obs_metadata=obs) cat.write_catalog(os.path.join(out_dir, gal_name), write_header=False, chunk_size=100000) cat = PhoSimCatalogSersic2D(disk_db, obs_metadata=obs) cat.write_catalog(os.path.join(out_dir, gal_name), write_header=False, write_mode='a', chunk_size=100000) cat = PhoSimCatalogZPoint(agn_db, obs_metadata=obs) cat.write_catalog(os.path.join(out_dir, agn_name), write_header=False, chunk_size=100000) for orig_name in (star_name, gal_name, agn_name): full_name = os.path.join(out_dir, orig_name) with open(full_name, 'r') as input_file: with gzip.open(full_name + '.gz', 'w') as output_file: output_file.writelines(input_file) os.unlink(full_name)
def testCompoundCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. This test uses CompoundInstanceCatalog """ # first, generate the catalog without a CompoundInstanceCatalog single_catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests','scratchSpace','phoSimTestCatalog_single.txt') testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) testBulge.write_catalog(single_catName) testDisk.write_catalog(single_catName, write_header=False, write_mode='a') testAgn.write_catalog(single_catName, write_header=False, write_mode='a') testStar.write_catalog(single_catName, write_header=False, write_mode='a') # now, generate the catalog using CompoundInstanceCatalog # # because the CompoundCatalogDBObject requires that database # connection parameters be set in the input CatalogDBObject # daughter class definitions, we have to declare dummy # CatalogDBObject daughter classes below class dummyDBbase(object): driver = 'sqlite' database = 'PhoSimTestDatabase.db' class dummyBulgeDB(dummyDBbase, testGalaxyBulgeDBObj): objid = 'dummy_bulge' class dummyDiskDB(dummyDBbase, testGalaxyDiskDBObj): objid = 'dummy_disk' class dummyAgnDB(dummyDBbase, testGalaxyAgnDBObj): objid = 'dummy_agn' class dummyStarDB(dummyDBbase, testStarsDBObj): objid = 'dummy_stars' compoundCatalog = CompoundInstanceCatalog([PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, PhoSimCatalogZPoint, PhoSimCatalogPoint], [dummyBulgeDB, dummyDiskDB, dummyAgnDB, dummyStarDB], obs_metadata=self.obs_metadata) self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3) compound_catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'phoSimTestCatalog_compound.txt') compoundCatalog.write_catalog(compound_catName) # verify that the compound catalog is what we expect self.verify_catalog(compound_catName) # verify that the two catalogs are equivalent with open(single_catName, 'r') as single_file: with open(compound_catName, 'r') as compound_file: single_lines = single_file.readlines() compound_lines = compound_file.readlines() for line in single_lines: self.assertIn(line, compound_lines) for line in compound_lines: self.assertIn(line, single_lines) if os.path.exists(compound_catName): os.unlink(compound_catName) if os.path.exists(single_catName): os.unlink(single_catName)