def testDefaultCustomCompoundCatalogDBObject(self): """ Test that CompoundInstanceCatalog can properly parse multiple CompoundCatalogDBobjects """ fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt') # negativeDecComopound_table2 should not come into play, since the # multiple queries are directed at table1 compoundCat = CompoundInstanceCatalog( [Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], compoundDBclass=[negativeDecCompound_table2, negativeRaCompound]) compoundCat.write_catalog(fileName) self.assertTrue(len(compoundCat._dbObjectGroupList) == 2) self.assertTrue(len(compoundCat._dbObjectGroupList[0]) == 2) self.assertTrue(len(compoundCat._dbObjectGroupList[1]) == 1) self.assertTrue(0 in compoundCat._dbObjectGroupList[0]) self.assertTrue(1 in compoundCat._dbObjectGroupList[0]) self.assertTrue(2 in compoundCat._dbObjectGroupList[1]) dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float)]) testData = numpy.genfromtxt(fileName, dtype=dtype) for line in testData: if line[0] < 2000: ix = line[0] - 1000 self.assertAlmostEqual(line[1], -1.0 * self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) elif line[0] < 3000: ix = line[0] - 2000 self.assertAlmostEqual( line[1], -2.0 * self.table1Control['ra'][ix] + self.table1Control['dra'][ix], 6) self.assertAlmostEqual( line[2], 2.0 * self.table1Control['dec'][ix] + self.table1Control['ddec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) else: ix = line[0] - 3000 self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) if os.path.exists(fileName): os.unlink(fileName)
def testCustomCompoundCatalogDBObject(self): """ Test that CompoundInstanceCatalog behaves properly when passed a custom CompoundCatalogDBObject """ fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt') compoundCat = CompoundInstanceCatalog( [Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], compoundDBclass=negativeRaCompound) compoundCat.write_catalog(fileName) self.assertTrue(len(compoundCat._dbObjectGroupList) == 2) self.assertTrue(len(compoundCat._dbObjectGroupList[0]) == 2) self.assertTrue(len(compoundCat._dbObjectGroupList[1]) == 1) self.assertTrue(0 in compoundCat._dbObjectGroupList[0]) self.assertTrue(1 in compoundCat._dbObjectGroupList[0]) self.assertTrue(2 in compoundCat._dbObjectGroupList[1]) dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float)]) testData = numpy.genfromtxt(fileName, dtype=dtype) for line in testData: if line[0] < 2000: ix = line[0] - 1000 self.assertAlmostEqual(line[1], -1.0 * self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) elif line[0] < 3000: ix = line[0] - 2000 self.assertAlmostEqual( line[1], -2.0 * self.table1Control['ra'][ix] + self.table1Control['dra'][ix], 6) self.assertAlmostEqual( line[2], 2.0 * self.table1Control['dec'][ix] + self.table1Control['ddec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) else: ix = line[0] - 3000 self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) if os.path.exists(fileName): os.unlink(fileName)
def testCustomCompoundCatalogDBObjectList(self): """ Test that CompoundInstanceCatalog behaves properly when there are two sets of multiple queries, one to table1, one to table2 """ fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt') compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3, Cat4], [table1DB1, table1DB2, table2DB1, table2DB2], compoundDBclass=[negativeRaCompound, negativeDecCompound_table2]) compoundCat.write_catalog(fileName) self.assertTrue(len(compoundCat._dbObjectGroupList)==2) self.assertTrue(len(compoundCat._dbObjectGroupList[0])==2) self.assertTrue(len(compoundCat._dbObjectGroupList[1])==2) self.assertTrue(0 in compoundCat._dbObjectGroupList[0]) self.assertTrue(1 in compoundCat._dbObjectGroupList[0]) self.assertTrue(2 in compoundCat._dbObjectGroupList[1]) self.assertTrue(3 in compoundCat._dbObjectGroupList[1]) dtype=numpy.dtype([ ('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float) ]) testData = numpy.genfromtxt(fileName, dtype=dtype) for line in testData: if line[0]<2000: ix = line[0]-1000 self.assertAlmostEqual(line[1], -1.0*self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) elif line[0]<3000: ix = line[0]-2000 self.assertAlmostEqual(line[1], -2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6) self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) elif line[0]<4000: ix = line[0]-3000 self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], -1.0*self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) else: ix = line[0]-4000 self.assertAlmostEqual(line[1], 2.0*self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], -2.0*self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) if os.path.exists(fileName): os.unlink(fileName)
def testDefaultCustomCompoundCatalogDBObject(self): """ Test that CompoundInstanceCatalog can properly parse multiple CompoundCatalogDBobjects """ fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt') # negativeDecComopound_table2 should not come into play, since the # multiple queries are directed at table1 compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], compoundDBclass=[negativeDecCompound_table2, negativeRaCompound]) compoundCat.write_catalog(fileName) self.assertTrue(len(compoundCat._dbObjectGroupList)==2) self.assertTrue(len(compoundCat._dbObjectGroupList[0])==2) self.assertTrue(len(compoundCat._dbObjectGroupList[1])==1) self.assertTrue(0 in compoundCat._dbObjectGroupList[0]) self.assertTrue(1 in compoundCat._dbObjectGroupList[0]) self.assertTrue(2 in compoundCat._dbObjectGroupList[1]) dtype=numpy.dtype([ ('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float) ]) testData = numpy.genfromtxt(fileName, dtype=dtype) for line in testData: if line[0]<2000: ix = line[0]-1000 self.assertAlmostEqual(line[1], -1.0*self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) elif line[0]<3000: ix = line[0]-2000 self.assertAlmostEqual(line[1], -2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6) self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) else: ix = line[0]-3000 self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) if os.path.exists(fileName): os.unlink(fileName)
def testCompoundCatalog(self): """ Test that a CompoundInstanceCatalog produces the expected output """ fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt') compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1]) compoundCat.write_catalog(fileName) self.assertTrue(len(compoundCat._dbObjectGroupList)==2) self.assertTrue(len(compoundCat._dbObjectGroupList[0])==2) self.assertTrue(len(compoundCat._dbObjectGroupList[1])==1) self.assertTrue(0 in compoundCat._dbObjectGroupList[0]) self.assertTrue(1 in compoundCat._dbObjectGroupList[0]) self.assertTrue(2 in compoundCat._dbObjectGroupList[1]) dtype=numpy.dtype([ ('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float) ]) testData = numpy.genfromtxt(fileName, dtype=dtype) for line in testData: if line[0]<2000: ix = line[0]-1000 self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) elif line[0]<3000: ix = line[0]-2000 self.assertAlmostEqual(line[1], 2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6) self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) else: ix = line[0]-3000 self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) if os.path.exists(fileName): os.unlink(fileName)
def write_catalog(self, outfile, obs_metadata, clobber=True): if clobber and os.path.isfile(outfile): os.remove(outfile) cat_list = [] for objid in self._instcatFactories: cat_list.append(self._instcatFactories[objid](obs_metadata)) if self.sprinkle: compoundDBclass = sprinklerCompound else: compoundDBclass = GalaxyTileCompoundObj while True: try: my_cat = CompoundInstanceCatalog(cat_list, obs_metadata=obs_metadata, compoundDBclass=compoundDBclass) break except RuntimeError: continue my_cat.write_catalog(outfile)
def write_catalog(self, outfile, obs_metadata, clobber=True): if clobber and os.path.isfile(outfile): os.remove(outfile) cat_list = [] for objid in self._instcatFactories: cat_list.append(self._instcatFactories[objid](obs_metadata)) if self.sprinkle: compoundDBclass = sprinklerCompound else: compoundDBclass = GalaxyTileCompoundObj while True: try: my_cat = CompoundInstanceCatalog( cat_list, obs_metadata=obs_metadata, compoundDBclass=compoundDBclass) break except RuntimeError: continue my_cat.write_catalog(outfile)
def testCompoundCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. This test uses CompoundInstanceCatalog """ testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) compoundCatalog = CompoundInstanceCatalog([testBulge, testDisk, testAgn, testStar], obs_metadata=self.obs_metadata) self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3) catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'phoSimTestCatalog_compound.txt') compoundCatalog.write_catalog(catName) testFile = open(catName,'r') testLines = testFile.readlines() testFile.close() controlLines = self.baseLineFile.readlines() for line in testLines: msg = '%s not in controlLines' % line self.assertTrue(line in controlLines, msg=msg) for line in controlLines: msg = '%s not in testLines' self.assertTrue(line in testLines, msg=msg) if os.path.exists(catName): os.unlink(catName)
def generatePhosimInput(mode='a', runobsHistID=None): if mode == 'a': filewrite = 'append' elif mode == 'c': filewrite = 'clobber' opsimDB = os.path.join('.','kraken_1042_sqlite.db') logfilename = 'run.log' if os.path.isfile(logfilename): if filewrite =='append': pass elif filewrite == 'clobber': with open('run.log', 'w') as f: f.write('obsHistID,status,timestamp\n') else: print('file exists and mode uncertain') exit() else: with open('run.log', 'w') as f: f.write('obsHistID,status,time\n') #you need to provide ObservationMetaDataGenerator with the connection #string to an OpSim output database. This is the connection string #to a test database that comes when you install CatSim. generator = ObservationMetaDataGenerator(database=opsimDB, driver='sqlite') obsHistIDList = numpy.genfromtxt('FirstSet_obsHistIDs.csv', delimiter=',', usecols=0) obsMetaDataResults = [] # Change the slicing in this line for the range of visits for obsHistID in obsHistIDList[600:700]: if runobsHistID is not None: obsHistID = runobsHistID obsMetaDataResults.append(generator.getObservationMetaData(obsHistID=obsHistID, fieldRA=(53, 54), fieldDec=(-29, -27), boundLength=0.3)[0]) starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars'] snmodel = SNObj() for obs_metadata in obsMetaDataResults: filename = "InstanceCatalogs / phosim_input_%s.txt" \ %(obs_metadata.phoSimMetaData['Opsim_obshistid'][0]) obs_metadata.phoSimMetaData['SIM_NSNAP'] = (1, numpy.dtype(int)) obs_metadata.phoSimMetaData['SIM_VISTIME'] = (30, numpy.dtype(float)) print('Starting Visit: ', obs_metadata.phoSimMetaData['Opsim_obshistid'][0]) compoundStarDBList = [MsStarObj, BhbStarObj, WdStarObj, RRLyStarObj, CepheidStarObj] compoundGalDBList = [GalaxyBulgeObj, GalaxyDiskObj, GalaxyAgnObj] compoundStarICList = [PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint] compoundGalICList = [PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, TwinklesCatalogZPoint] snphosim = PhoSimCatalogSN(db_obj=snmodel, obs_metadata=obs_metadata, column_outputs=['EBV']) snphosim.writeSedFile = True snphosim.suppressDimSN = True snphosim.prefix = 'spectra_files/' while True: try: starCat = CompoundInstanceCatalog(compoundStarICList, compoundStarDBList, obs_metadata=obs_metadata, constraint='gmag > 11.', compoundDBclass=sprinklerCompound) starCat.write_catalog(filename, chunk_size=10000) galCat = CompoundInstanceCatalog(compoundGalICList, compoundGalDBList, obs_metadata=obs_metadata, # constraint='g_ab > 11.', compoundDBclass=sprinklerCompound) galCat.write_catalog(filename, write_mode='a', write_header=False, chunk_size=10000) snphosim.write_catalog(filename, write_header=False, write_mode='a', chunk_size=10000) if runobsHistID is not None: print('Done doing requested obsHistID') sys.exit() with open(logfilename, 'a') as f: f.write('{0:d},DONE,{1:3.6f}\n'.format(obs_metadata.phoSimMetaData['Opsim_obshistid'][0], time.time())) break except RuntimeError: continue print("Finished Writing Visit: ", obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
def testCompoundCatalog(self): """ This test writes a PhoSim input catalog and compares it, one line at a time to a previously written catalog that should be identical. This test uses CompoundInstanceCatalog """ # first, generate the catalog without a CompoundInstanceCatalog single_catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests','scratchSpace','phoSimTestCatalog_single.txt') testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata) testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata) testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata) testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata) testBulge.write_catalog(single_catName) testDisk.write_catalog(single_catName, write_header=False, write_mode='a') testAgn.write_catalog(single_catName, write_header=False, write_mode='a') testStar.write_catalog(single_catName, write_header=False, write_mode='a') # now, generate the catalog using CompoundInstanceCatalog # # because the CompoundCatalogDBObject requires that database # connection parameters be set in the input CatalogDBObject # daughter class definitions, we have to declare dummy # CatalogDBObject daughter classes below class dummyDBbase(object): driver = 'sqlite' database = 'PhoSimTestDatabase.db' class dummyBulgeDB(dummyDBbase, testGalaxyBulgeDBObj): objid = 'dummy_bulge' class dummyDiskDB(dummyDBbase, testGalaxyDiskDBObj): objid = 'dummy_disk' class dummyAgnDB(dummyDBbase, testGalaxyAgnDBObj): objid = 'dummy_agn' class dummyStarDB(dummyDBbase, testStarsDBObj): objid = 'dummy_stars' compoundCatalog = CompoundInstanceCatalog([PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, PhoSimCatalogZPoint, PhoSimCatalogPoint], [dummyBulgeDB, dummyDiskDB, dummyAgnDB, dummyStarDB], obs_metadata=self.obs_metadata) self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3) compound_catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'phoSimTestCatalog_compound.txt') compoundCatalog.write_catalog(compound_catName) # verify that the compound catalog is what we expect self.verify_catalog(compound_catName) # verify that the two catalogs are equivalent with open(single_catName, 'r') as single_file: with open(compound_catName, 'r') as compound_file: single_lines = single_file.readlines() compound_lines = compound_file.readlines() for line in single_lines: self.assertIn(line, compound_lines) for line in compound_lines: self.assertIn(line, single_lines) if os.path.exists(compound_catName): os.unlink(compound_catName) if os.path.exists(single_catName): os.unlink(single_catName)
def testObservationMetaDataAndConstraint(self): """ Test that CompoundInstanceCatalog handles ObservationMetaData and a constraint properly """ fileName = os.path.join(self.baseDir, 'compound_obs_metadata_test_cat.txt') obs = ObservationMetaData(pointingRA = 180.0, pointingDec = 0.0, boundType = 'box', boundLength = (80.0, 25.0), mjd=53850.0) compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], obs_metadata=obs, constraint='mag>20.0') compoundCat.write_catalog(fileName) dtype=numpy.dtype([ ('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float) ]) testData = numpy.genfromtxt(fileName, dtype=dtype) table1_good_rows = [] table2_good_rows = [] for line in testData: if line[0]<2000: ix = line[0]-1000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['ra'][ix]>100.0 and self.table1Control['ra'][ix]<260.0) self.assertTrue(self.table1Control['dec'][ix]>-25.0 and self.table1Control['dec'][ix]<25.0) self.assertTrue(self.table1Control['mag'][ix]>20.0) elif line[0]<3000: ix = line[0]-2000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual(line[1], 2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6) self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['ra'][ix]>100.0 and self.table1Control['ra'][ix]<260.0) self.assertTrue(self.table1Control['dec'][ix]>-25.0 and self.table1Control['dec'][ix]<25.0) self.assertTrue(self.table1Control['mag'][ix]>20.0) else: ix = line[0]-3000 if ix not in table2_good_rows: table2_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) self.assertTrue(self.table2Control['ra'][ix]>100.0 and self.table2Control['ra'][ix]<260.0) self.assertTrue(self.table2Control['dec'][ix]>-25.0 and self.table2Control['dec'][ix]<25.0) self.assertTrue(self.table2Control['mag'][ix]>20.0) table1_bad_rows = [ix for ix in range(self.table1Control.shape[0]) if ix not in table1_good_rows] table2_bad_rows = [ix for ix in range(self.table2Control.shape[0]) if ix not in table2_good_rows] in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \ for rr, dd, mm in zip(self.table1Control['ra'][table1_bad_rows], self.table1Control['dec'][table1_bad_rows], self.table1Control['mag'][table1_bad_rows])] self.assertFalse(True in in_bounds) in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \ for rr, dd, mm in zip(self.table2Control['ra'][table2_bad_rows], self.table2Control['dec'][table2_bad_rows], self.table2Control['mag'][table2_bad_rows])] self.assertFalse(True in in_bounds) self.assertTrue(len(table1_good_rows)>0) self.assertTrue(len(table2_good_rows)>0) self.assertTrue(len(table1_bad_rows)>0) self.assertTrue(len(table2_bad_rows)>0) self.assertEqual(len(table1_good_rows)+len(table1_bad_rows), self.table1Control.shape[0]) self.assertEqual(len(table2_good_rows)+len(table2_bad_rows), self.table2Control.shape[0]) if os.path.exists(fileName): os.unlink(fileName)
def testConstraint(self): """ Test that CompoundInstanceCatalog handles constraint properly """ fileName = os.path.join(self.baseDir, 'compound_constraint_test_cat.txt') compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], constraint='mag>20.0') compoundCat.write_catalog(fileName) dtype=numpy.dtype([ ('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float) ]) testData = numpy.genfromtxt(fileName, dtype=dtype) table1_good_rows = [] table2_good_rows = [] for line in testData: if line[0]<2000: ix = line[0]-1000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['mag'][ix]>20.0) elif line[0]<3000: ix = line[0]-2000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual(line[1], 2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6) self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6) self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['mag'][ix]>20.0) else: ix = line[0]-3000 if ix not in table2_good_rows: table2_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) self.assertTrue(self.table2Control['mag'][ix]>20.0) table1_bad_rows = [ix for ix in range(self.table1Control.shape[0]) if ix not in table1_good_rows] table2_bad_rows = [ix for ix in range(self.table2Control.shape[0]) if ix not in table2_good_rows] in_bounds = [mm>20.0 for mm in self.table1Control['mag'][table1_bad_rows]] self.assertFalse(True in in_bounds) in_bounds = [mm>20.0 for mm in self.table2Control['mag'][table2_bad_rows]] self.assertFalse(True in in_bounds) self.assertTrue(len(table1_good_rows)>0) self.assertTrue(len(table2_good_rows)>0) self.assertTrue(len(table1_bad_rows)>0) self.assertTrue(len(table2_bad_rows)>0) self.assertEqual(len(table1_good_rows)+len(table1_bad_rows), self.table1Control.shape[0]) self.assertEqual(len(table2_good_rows)+len(table2_bad_rows), self.table2Control.shape[0]) if os.path.exists(fileName): os.unlink(fileName)
def testObservationMetaDataAndConstraint(self): """ Test that CompoundInstanceCatalog handles ObservationMetaData and a constraint properly """ fileName = os.path.join(self.baseDir, 'compound_obs_metadata_test_cat.txt') obs = ObservationMetaData(pointingRA=180.0, pointingDec=0.0, boundType='box', boundLength=(80.0, 25.0), mjd=53850.0) compoundCat = CompoundInstanceCatalog( [Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], obs_metadata=obs, constraint='mag>20.0') compoundCat.write_catalog(fileName) dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float)]) testData = numpy.genfromtxt(fileName, dtype=dtype) table1_good_rows = [] table2_good_rows = [] for line in testData: if line[0] < 2000: ix = line[0] - 1000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['ra'][ix] > 100.0 and self.table1Control['ra'][ix] < 260.0) self.assertTrue(self.table1Control['dec'][ix] > -25.0 and self.table1Control['dec'][ix] < 25.0) self.assertTrue(self.table1Control['mag'][ix] > 20.0) elif line[0] < 3000: ix = line[0] - 2000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual( line[1], 2.0 * self.table1Control['ra'][ix] + self.table1Control['dra'][ix], 6) self.assertAlmostEqual( line[2], 2.0 * self.table1Control['dec'][ix] + self.table1Control['ddec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['ra'][ix] > 100.0 and self.table1Control['ra'][ix] < 260.0) self.assertTrue(self.table1Control['dec'][ix] > -25.0 and self.table1Control['dec'][ix] < 25.0) self.assertTrue(self.table1Control['mag'][ix] > 20.0) else: ix = line[0] - 3000 if ix not in table2_good_rows: table2_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) self.assertTrue(self.table2Control['ra'][ix] > 100.0 and self.table2Control['ra'][ix] < 260.0) self.assertTrue(self.table2Control['dec'][ix] > -25.0 and self.table2Control['dec'][ix] < 25.0) self.assertTrue(self.table2Control['mag'][ix] > 20.0) table1_bad_rows = [ ix for ix in range(self.table1Control.shape[0]) if ix not in table1_good_rows ] table2_bad_rows = [ ix for ix in range(self.table2Control.shape[0]) if ix not in table2_good_rows ] in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \ for rr, dd, mm in zip(self.table1Control['ra'][table1_bad_rows], self.table1Control['dec'][table1_bad_rows], self.table1Control['mag'][table1_bad_rows])] self.assertFalse(True in in_bounds) in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \ for rr, dd, mm in zip(self.table2Control['ra'][table2_bad_rows], self.table2Control['dec'][table2_bad_rows], self.table2Control['mag'][table2_bad_rows])] self.assertFalse(True in in_bounds) self.assertTrue(len(table1_good_rows) > 0) self.assertTrue(len(table2_good_rows) > 0) self.assertTrue(len(table1_bad_rows) > 0) self.assertTrue(len(table2_bad_rows) > 0) self.assertEqual( len(table1_good_rows) + len(table1_bad_rows), self.table1Control.shape[0]) self.assertEqual( len(table2_good_rows) + len(table2_bad_rows), self.table2Control.shape[0]) if os.path.exists(fileName): os.unlink(fileName)
def testConstraint(self): """ Test that CompoundInstanceCatalog handles constraint properly """ fileName = os.path.join(self.baseDir, 'compound_constraint_test_cat.txt') compoundCat = CompoundInstanceCatalog( [Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1], constraint='mag>20.0') compoundCat.write_catalog(fileName) dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float), ('decObs', numpy.float), ('final_mag', numpy.float)]) testData = numpy.genfromtxt(fileName, dtype=dtype) table1_good_rows = [] table2_good_rows = [] for line in testData: if line[0] < 2000: ix = line[0] - 1000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['mag'][ix] > 20.0) elif line[0] < 3000: ix = line[0] - 2000 if ix not in table1_good_rows: table1_good_rows.append(ix) self.assertAlmostEqual( line[1], 2.0 * self.table1Control['ra'][ix] + self.table1Control['dra'][ix], 6) self.assertAlmostEqual( line[2], 2.0 * self.table1Control['dec'][ix] + self.table1Control['ddec'][ix], 6) self.assertAlmostEqual( line[3], self.table1Control['mag'][ix] + self.table1Control['dmag'][ix], 6) self.assertTrue(self.table1Control['mag'][ix] > 20.0) else: ix = line[0] - 3000 if ix not in table2_good_rows: table2_good_rows.append(ix) self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6) self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6) self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6) self.assertTrue(self.table2Control['mag'][ix] > 20.0) table1_bad_rows = [ ix for ix in range(self.table1Control.shape[0]) if ix not in table1_good_rows ] table2_bad_rows = [ ix for ix in range(self.table2Control.shape[0]) if ix not in table2_good_rows ] in_bounds = [ mm > 20.0 for mm in self.table1Control['mag'][table1_bad_rows] ] self.assertFalse(True in in_bounds) in_bounds = [ mm > 20.0 for mm in self.table2Control['mag'][table2_bad_rows] ] self.assertFalse(True in in_bounds) self.assertTrue(len(table1_good_rows) > 0) self.assertTrue(len(table2_good_rows) > 0) self.assertTrue(len(table1_bad_rows) > 0) self.assertTrue(len(table2_bad_rows) > 0) self.assertEqual( len(table1_good_rows) + len(table1_bad_rows), self.table1Control.shape[0]) self.assertEqual( len(table2_good_rows) + len(table2_bad_rows), self.table2Control.shape[0]) if os.path.exists(fileName): os.unlink(fileName)
def testGalaxyCatalog(self): """ Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks, and agns using both the 'old fashioned way' (one catalog at a time), and using CompoundInstanceCatalog """ controlFileName = os.path.join(self.baseDir, 'gal_compound_control.txt') testFileName = os.path.join(self.baseDir, 'gal_compound_test.txt') if os.path.exists(controlFileName): os.unlink(controlFileName) if os.path.exists(testFileName): os.unlink(testFileName) obs = ObservationMetaData(unrefractedRA=25.0, unrefractedDec=-45.0, boundType='circle', boundLength=0.05) dbBulge = GalaxyBulgeObj() dbDisk = GalaxyDiskObj() dbAgn = GalaxyAgnObj() catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs) catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs) catAgn = AgnCatalog(dbAgn, obs_metadata=obs) catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000) catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000) # You need to reinstantiate the catalogs because the process of writing them # above stripped galtileid from their _active_columns, which is the only way # CompoundInstanceCatalog can know that it needs to worry about galtileid catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs) catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs) catAgn = AgnCatalog(dbAgn, obs_metadata=obs) totalCat = CompoundInstanceCatalog([catBulge, catDisk, catAgn], obs_metadata=obs, compoundDBclass=GalaxyTileCompoundObj) totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000) controlFile = open(controlFileName, 'r') control = controlFile.readlines() controlFile.close() testFile = open(testFileName, 'r') test = testFile.readlines() testFile.close() for line in control: self.assertTrue(line in test) for line in test: self.assertTrue(line in control) if os.path.exists(controlFileName): os.unlink(controlFileName) if os.path.exists(testFileName): os.unlink(testFileName)
def generatePhosimInput(mode='a', runobsHistID=None): if mode == 'a': filewrite = 'append' elif mode == 'c': filewrite = 'clobber' opsimDB = os.path.join('.', 'kraken_1042_sqlite.db') logfilename = 'run.log' if os.path.isfile(logfilename): if filewrite == 'append': pass elif filewrite == 'clobber': with open('run.log', 'w') as f: f.write('obsHistID,status,timestamp\n') else: print('file exists and mode uncertain') exit() else: with open('run.log', 'w') as f: f.write('obsHistID,status,time\n') #you need to provide ObservationMetaDataGenerator with the connection #string to an OpSim output database. This is the connection string #to a test database that comes when you install CatSim. generator = ObservationMetaDataGenerator(database=opsimDB, driver='sqlite') obsHistIDList = numpy.genfromtxt('FirstSet_obsHistIDs.csv', delimiter=',', usecols=0) obsMetaDataResults = [] # Change the slicing in this line for the range of visits for obsHistID in obsHistIDList[600:700]: if runobsHistID is not None: obsHistID = runobsHistID obsMetaDataResults.append( generator.getObservationMetaData(obsHistID=obsHistID, fieldRA=(53, 54), fieldDec=(-29, -27), boundLength=0.3)[0]) starObjNames = [ 'msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars' ] snmodel = SNObj() for obs_metadata in obsMetaDataResults: filename = "InstanceCatalogs / phosim_input_%s.txt" \ %(obs_metadata.phoSimMetaData['Opsim_obshistid'][0]) obs_metadata.phoSimMetaData['SIM_NSNAP'] = (1, numpy.dtype(int)) obs_metadata.phoSimMetaData['SIM_VISTIME'] = (30, numpy.dtype(float)) print('Starting Visit: ', obs_metadata.phoSimMetaData['Opsim_obshistid'][0]) compoundStarDBList = [ MsStarObj, BhbStarObj, WdStarObj, RRLyStarObj, CepheidStarObj ] compoundGalDBList = [GalaxyBulgeObj, GalaxyDiskObj, GalaxyAgnObj] compoundStarICList = [ PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint ] compoundGalICList = [ PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, TwinklesCatalogZPoint ] snphosim = PhoSimCatalogSN(db_obj=snmodel, obs_metadata=obs_metadata, column_outputs=['EBV']) snphosim.writeSedFile = True snphosim.suppressDimSN = True snphosim.prefix = 'spectra_files/' while True: try: starCat = CompoundInstanceCatalog( compoundStarICList, compoundStarDBList, obs_metadata=obs_metadata, constraint='gmag > 11.', compoundDBclass=sprinklerCompound) starCat.write_catalog(filename, chunk_size=10000) galCat = CompoundInstanceCatalog( compoundGalICList, compoundGalDBList, obs_metadata=obs_metadata, # constraint='g_ab > 11.', compoundDBclass=sprinklerCompound) galCat.write_catalog(filename, write_mode='a', write_header=False, chunk_size=10000) snphosim.write_catalog(filename, write_header=False, write_mode='a', chunk_size=10000) if runobsHistID is not None: print('Done doing requested obsHistID') sys.exit() with open(logfilename, 'a') as f: f.write('{0:d},DONE,{1:3.6f}\n'.format( obs_metadata.phoSimMetaData['Opsim_obshistid'][0], time.time())) break except RuntimeError: continue print("Finished Writing Visit: ", obs_metadata.phoSimMetaData['Opsim_obshistid'][0])