def testDefaultCustomCompoundCatalogDBObject(self):
"""
Test that CompoundInstanceCatalog can properly parse multiple CompoundCatalogDBobjects
"""
fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt')
# negativeDecComopound_table2 should not come into play, since the
# multiple queries are directed at table1
compoundCat = CompoundInstanceCatalog(
[Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1],
compoundDBclass=[negativeDecCompound_table2, negativeRaCompound])
compoundCat.write_catalog(fileName)
self.assertTrue(len(compoundCat._dbObjectGroupList) == 2)
self.assertTrue(len(compoundCat._dbObjectGroupList[0]) == 2)
self.assertTrue(len(compoundCat._dbObjectGroupList[1]) == 1)
self.assertTrue(0 in compoundCat._dbObjectGroupList[0])
self.assertTrue(1 in compoundCat._dbObjectGroupList[0])
self.assertTrue(2 in compoundCat._dbObjectGroupList[1])
dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)])
testData = numpy.genfromtxt(fileName, dtype=dtype)
for line in testData:
if line[0] < 2000:
ix = line[0] - 1000
self.assertAlmostEqual(line[1],
-1.0 * self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix],
6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
elif line[0] < 3000:
ix = line[0] - 2000
self.assertAlmostEqual(
line[1], -2.0 * self.table1Control['ra'][ix] +
self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(
line[2], 2.0 * self.table1Control['dec'][ix] +
self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
else:
ix = line[0] - 3000
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix],
6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix],
6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix],
6)
if os.path.exists(fileName):
os.unlink(fileName)
def testCustomCompoundCatalogDBObject(self):
"""
Test that CompoundInstanceCatalog behaves properly when passed a
custom CompoundCatalogDBObject
"""
fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt')
compoundCat = CompoundInstanceCatalog(
[Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1],
compoundDBclass=negativeRaCompound)
compoundCat.write_catalog(fileName)
self.assertTrue(len(compoundCat._dbObjectGroupList) == 2)
self.assertTrue(len(compoundCat._dbObjectGroupList[0]) == 2)
self.assertTrue(len(compoundCat._dbObjectGroupList[1]) == 1)
self.assertTrue(0 in compoundCat._dbObjectGroupList[0])
self.assertTrue(1 in compoundCat._dbObjectGroupList[0])
self.assertTrue(2 in compoundCat._dbObjectGroupList[1])
dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)])
testData = numpy.genfromtxt(fileName, dtype=dtype)
for line in testData:
if line[0] < 2000:
ix = line[0] - 1000
self.assertAlmostEqual(line[1],
-1.0 * self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix],
6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
elif line[0] < 3000:
ix = line[0] - 2000
self.assertAlmostEqual(
line[1], -2.0 * self.table1Control['ra'][ix] +
self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(
line[2], 2.0 * self.table1Control['dec'][ix] +
self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
else:
ix = line[0] - 3000
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix],
6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix],
6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix],
6)
if os.path.exists(fileName):
os.unlink(fileName)
def testCustomCompoundCatalogDBObjectList(self):
"""
Test that CompoundInstanceCatalog behaves properly when there are
two sets of multiple queries, one to table1, one to table2
"""
fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt')
compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3, Cat4],
[table1DB1, table1DB2, table2DB1, table2DB2],
compoundDBclass=[negativeRaCompound, negativeDecCompound_table2])
compoundCat.write_catalog(fileName)
self.assertTrue(len(compoundCat._dbObjectGroupList)==2)
self.assertTrue(len(compoundCat._dbObjectGroupList[0])==2)
self.assertTrue(len(compoundCat._dbObjectGroupList[1])==2)
self.assertTrue(0 in compoundCat._dbObjectGroupList[0])
self.assertTrue(1 in compoundCat._dbObjectGroupList[0])
self.assertTrue(2 in compoundCat._dbObjectGroupList[1])
self.assertTrue(3 in compoundCat._dbObjectGroupList[1])
dtype=numpy.dtype([
('id', numpy.int),
('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)
])
testData = numpy.genfromtxt(fileName, dtype=dtype)
for line in testData:
if line[0]<2000:
ix = line[0]-1000
self.assertAlmostEqual(line[1], -1.0*self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
elif line[0]<3000:
ix = line[0]-2000
self.assertAlmostEqual(line[1], -2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
elif line[0]<4000:
ix = line[0]-3000
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], -1.0*self.table2Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6)
else:
ix = line[0]-4000
self.assertAlmostEqual(line[1], 2.0*self.table2Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], -2.0*self.table2Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6)
if os.path.exists(fileName):
os.unlink(fileName)
def testDefaultCustomCompoundCatalogDBObject(self):
"""
Test that CompoundInstanceCatalog can properly parse multiple CompoundCatalogDBobjects
"""
fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt')
# negativeDecComopound_table2 should not come into play, since the
# multiple queries are directed at table1
compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3],
[table1DB1, table1DB2, table2DB1],
compoundDBclass=[negativeDecCompound_table2, negativeRaCompound])
compoundCat.write_catalog(fileName)
self.assertTrue(len(compoundCat._dbObjectGroupList)==2)
self.assertTrue(len(compoundCat._dbObjectGroupList[0])==2)
self.assertTrue(len(compoundCat._dbObjectGroupList[1])==1)
self.assertTrue(0 in compoundCat._dbObjectGroupList[0])
self.assertTrue(1 in compoundCat._dbObjectGroupList[0])
self.assertTrue(2 in compoundCat._dbObjectGroupList[1])
dtype=numpy.dtype([
('id', numpy.int),
('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)
])
testData = numpy.genfromtxt(fileName, dtype=dtype)
for line in testData:
if line[0]<2000:
ix = line[0]-1000
self.assertAlmostEqual(line[1], -1.0*self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
elif line[0]<3000:
ix = line[0]-2000
self.assertAlmostEqual(line[1], -2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
else:
ix = line[0]-3000
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6)
if os.path.exists(fileName):
os.unlink(fileName)
def testCompoundCatalog(self):
"""
Test that a CompoundInstanceCatalog produces the expected output
"""
fileName = os.path.join(self.baseDir, 'simplest_compound_catalog.txt')
compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1])
compoundCat.write_catalog(fileName)
self.assertTrue(len(compoundCat._dbObjectGroupList)==2)
self.assertTrue(len(compoundCat._dbObjectGroupList[0])==2)
self.assertTrue(len(compoundCat._dbObjectGroupList[1])==1)
self.assertTrue(0 in compoundCat._dbObjectGroupList[0])
self.assertTrue(1 in compoundCat._dbObjectGroupList[0])
self.assertTrue(2 in compoundCat._dbObjectGroupList[1])
dtype=numpy.dtype([
('id', numpy.int),
('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)
])
testData = numpy.genfromtxt(fileName, dtype=dtype)
for line in testData:
if line[0]<2000:
ix = line[0]-1000
self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
elif line[0]<3000:
ix = line[0]-2000
self.assertAlmostEqual(line[1], 2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
else:
ix = line[0]-3000
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6)
if os.path.exists(fileName):
os.unlink(fileName)
def write_catalog(self, outfile, obs_metadata, clobber=True):
if clobber and os.path.isfile(outfile):
os.remove(outfile)
cat_list = []
for objid in self._instcatFactories:
cat_list.append(self._instcatFactories[objid](obs_metadata))
if self.sprinkle:
compoundDBclass = sprinklerCompound
else:
compoundDBclass = GalaxyTileCompoundObj
while True:
try:
my_cat = CompoundInstanceCatalog(cat_list, obs_metadata=obs_metadata, compoundDBclass=compoundDBclass)
break
except RuntimeError:
continue
my_cat.write_catalog(outfile)
def write_catalog(self, outfile, obs_metadata, clobber=True):
if clobber and os.path.isfile(outfile):
os.remove(outfile)
cat_list = []
for objid in self._instcatFactories:
cat_list.append(self._instcatFactories[objid](obs_metadata))
if self.sprinkle:
compoundDBclass = sprinklerCompound
else:
compoundDBclass = GalaxyTileCompoundObj
while True:
try:
my_cat = CompoundInstanceCatalog(
cat_list,
obs_metadata=obs_metadata,
compoundDBclass=compoundDBclass)
break
except RuntimeError:
continue
my_cat.write_catalog(outfile)
def testCompoundCatalog(self):
"""
This test writes a PhoSim input catalog and compares it, one line at a time
to a previously written catalog that should be identical.
This test uses CompoundInstanceCatalog
"""
testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata)
testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata)
testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata)
testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata)
compoundCatalog = CompoundInstanceCatalog([testBulge, testDisk, testAgn, testStar],
obs_metadata=self.obs_metadata)
self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3)
catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace',
'phoSimTestCatalog_compound.txt')
compoundCatalog.write_catalog(catName)
testFile = open(catName,'r')
testLines = testFile.readlines()
testFile.close()
controlLines = self.baseLineFile.readlines()
for line in testLines:
msg = '%s not in controlLines' % line
self.assertTrue(line in controlLines, msg=msg)
for line in controlLines:
msg = '%s not in testLines'
self.assertTrue(line in testLines, msg=msg)
if os.path.exists(catName):
os.unlink(catName)
def generatePhosimInput(mode='a', runobsHistID=None):
if mode == 'a':
filewrite = 'append'
elif mode == 'c':
filewrite = 'clobber'
opsimDB = os.path.join('.','kraken_1042_sqlite.db')
logfilename = 'run.log'
if os.path.isfile(logfilename):
if filewrite =='append':
pass
elif filewrite == 'clobber':
with open('run.log', 'w') as f:
f.write('obsHistID,status,timestamp\n')
else:
print('file exists and mode uncertain')
exit()
else:
with open('run.log', 'w') as f:
f.write('obsHistID,status,time\n')
#you need to provide ObservationMetaDataGenerator with the connection
#string to an OpSim output database. This is the connection string
#to a test database that comes when you install CatSim.
generator = ObservationMetaDataGenerator(database=opsimDB, driver='sqlite')
obsHistIDList = numpy.genfromtxt('FirstSet_obsHistIDs.csv', delimiter=',', usecols=0)
obsMetaDataResults = []
# Change the slicing in this line for the range of visits
for obsHistID in obsHistIDList[600:700]:
if runobsHistID is not None:
obsHistID = runobsHistID
obsMetaDataResults.append(generator.getObservationMetaData(obsHistID=obsHistID,
fieldRA=(53, 54), fieldDec=(-29, -27),
boundLength=0.3)[0])
starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars']
snmodel = SNObj()
for obs_metadata in obsMetaDataResults:
filename = "InstanceCatalogs / phosim_input_%s.txt" \
%(obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
obs_metadata.phoSimMetaData['SIM_NSNAP'] = (1, numpy.dtype(int))
obs_metadata.phoSimMetaData['SIM_VISTIME'] = (30, numpy.dtype(float))
print('Starting Visit: ',
obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
compoundStarDBList = [MsStarObj, BhbStarObj, WdStarObj, RRLyStarObj,
CepheidStarObj]
compoundGalDBList = [GalaxyBulgeObj, GalaxyDiskObj, GalaxyAgnObj]
compoundStarICList = [PhoSimCatalogPoint, PhoSimCatalogPoint,
PhoSimCatalogPoint, PhoSimCatalogPoint,
PhoSimCatalogPoint]
compoundGalICList = [PhoSimCatalogSersic2D, PhoSimCatalogSersic2D,
TwinklesCatalogZPoint]
snphosim = PhoSimCatalogSN(db_obj=snmodel,
obs_metadata=obs_metadata,
column_outputs=['EBV'])
snphosim.writeSedFile = True
snphosim.suppressDimSN = True
snphosim.prefix = 'spectra_files/'
while True:
try:
starCat = CompoundInstanceCatalog(compoundStarICList,
compoundStarDBList,
obs_metadata=obs_metadata,
constraint='gmag > 11.',
compoundDBclass=sprinklerCompound)
starCat.write_catalog(filename, chunk_size=10000)
galCat = CompoundInstanceCatalog(compoundGalICList,
compoundGalDBList,
obs_metadata=obs_metadata,
# constraint='g_ab > 11.',
compoundDBclass=sprinklerCompound)
galCat.write_catalog(filename, write_mode='a',
write_header=False, chunk_size=10000)
snphosim.write_catalog(filename, write_header=False,
write_mode='a', chunk_size=10000)
if runobsHistID is not None:
print('Done doing requested obsHistID')
sys.exit()
with open(logfilename, 'a') as f:
f.write('{0:d},DONE,{1:3.6f}\n'.format(obs_metadata.phoSimMetaData['Opsim_obshistid'][0], time.time()))
break
except RuntimeError:
continue
print("Finished Writing Visit: ", obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
def testCompoundCatalog(self):
"""
This test writes a PhoSim input catalog and compares it, one line at a time
to a previously written catalog that should be identical.
This test uses CompoundInstanceCatalog
"""
# first, generate the catalog without a CompoundInstanceCatalog
single_catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'),
'tests','scratchSpace','phoSimTestCatalog_single.txt')
testBulge = PhoSimCatalogSersic2D(self.bulgeDB, obs_metadata = self.obs_metadata)
testDisk = PhoSimCatalogSersic2D(self.diskDB, obs_metadata = self.obs_metadata)
testAgn = PhoSimCatalogZPoint(self.agnDB, obs_metadata = self.obs_metadata)
testStar = PhoSimCatalogPoint(self.starDB, obs_metadata = self.obs_metadata)
testBulge.write_catalog(single_catName)
testDisk.write_catalog(single_catName, write_header=False, write_mode='a')
testAgn.write_catalog(single_catName, write_header=False, write_mode='a')
testStar.write_catalog(single_catName, write_header=False, write_mode='a')
# now, generate the catalog using CompoundInstanceCatalog
#
# because the CompoundCatalogDBObject requires that database
# connection parameters be set in the input CatalogDBObject
# daughter class definitions, we have to declare dummy
# CatalogDBObject daughter classes below
class dummyDBbase(object):
driver = 'sqlite'
database = 'PhoSimTestDatabase.db'
class dummyBulgeDB(dummyDBbase, testGalaxyBulgeDBObj):
objid = 'dummy_bulge'
class dummyDiskDB(dummyDBbase, testGalaxyDiskDBObj):
objid = 'dummy_disk'
class dummyAgnDB(dummyDBbase, testGalaxyAgnDBObj):
objid = 'dummy_agn'
class dummyStarDB(dummyDBbase, testStarsDBObj):
objid = 'dummy_stars'
compoundCatalog = CompoundInstanceCatalog([PhoSimCatalogSersic2D, PhoSimCatalogSersic2D,
PhoSimCatalogZPoint, PhoSimCatalogPoint],
[dummyBulgeDB, dummyDiskDB, dummyAgnDB, dummyStarDB],
obs_metadata=self.obs_metadata)
self.assertEqual(len(compoundCatalog._dbObjectGroupList[0]), 3)
compound_catName = os.path.join(lsst.utils.getPackageDir('sims_catUtils'), 'tests', 'scratchSpace',
'phoSimTestCatalog_compound.txt')
compoundCatalog.write_catalog(compound_catName)
# verify that the compound catalog is what we expect
self.verify_catalog(compound_catName)
# verify that the two catalogs are equivalent
with open(single_catName, 'r') as single_file:
with open(compound_catName, 'r') as compound_file:
single_lines = single_file.readlines()
compound_lines = compound_file.readlines()
for line in single_lines:
self.assertIn(line, compound_lines)
for line in compound_lines:
self.assertIn(line, single_lines)
if os.path.exists(compound_catName):
os.unlink(compound_catName)
if os.path.exists(single_catName):
os.unlink(single_catName)
def testObservationMetaDataAndConstraint(self):
"""
Test that CompoundInstanceCatalog handles ObservationMetaData
and a constraint properly
"""
fileName = os.path.join(self.baseDir, 'compound_obs_metadata_test_cat.txt')
obs = ObservationMetaData(pointingRA = 180.0,
pointingDec = 0.0,
boundType = 'box',
boundLength = (80.0, 25.0),
mjd=53850.0)
compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1],
obs_metadata=obs,
constraint='mag>20.0')
compoundCat.write_catalog(fileName)
dtype=numpy.dtype([
('id', numpy.int),
('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)
])
testData = numpy.genfromtxt(fileName, dtype=dtype)
table1_good_rows = []
table2_good_rows = []
for line in testData:
if line[0]<2000:
ix = line[0]-1000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['ra'][ix]>100.0 and self.table1Control['ra'][ix]<260.0)
self.assertTrue(self.table1Control['dec'][ix]>-25.0 and self.table1Control['dec'][ix]<25.0)
self.assertTrue(self.table1Control['mag'][ix]>20.0)
elif line[0]<3000:
ix = line[0]-2000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(line[1], 2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['ra'][ix]>100.0 and self.table1Control['ra'][ix]<260.0)
self.assertTrue(self.table1Control['dec'][ix]>-25.0 and self.table1Control['dec'][ix]<25.0)
self.assertTrue(self.table1Control['mag'][ix]>20.0)
else:
ix = line[0]-3000
if ix not in table2_good_rows:
table2_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6)
self.assertTrue(self.table2Control['ra'][ix]>100.0 and self.table2Control['ra'][ix]<260.0)
self.assertTrue(self.table2Control['dec'][ix]>-25.0 and self.table2Control['dec'][ix]<25.0)
self.assertTrue(self.table2Control['mag'][ix]>20.0)
table1_bad_rows = [ix for ix in range(self.table1Control.shape[0]) if ix not in table1_good_rows]
table2_bad_rows = [ix for ix in range(self.table2Control.shape[0]) if ix not in table2_good_rows]
in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \
for rr, dd, mm in zip(self.table1Control['ra'][table1_bad_rows],
self.table1Control['dec'][table1_bad_rows],
self.table1Control['mag'][table1_bad_rows])]
self.assertFalse(True in in_bounds)
in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \
for rr, dd, mm in zip(self.table2Control['ra'][table2_bad_rows],
self.table2Control['dec'][table2_bad_rows],
self.table2Control['mag'][table2_bad_rows])]
self.assertFalse(True in in_bounds)
self.assertTrue(len(table1_good_rows)>0)
self.assertTrue(len(table2_good_rows)>0)
self.assertTrue(len(table1_bad_rows)>0)
self.assertTrue(len(table2_bad_rows)>0)
self.assertEqual(len(table1_good_rows)+len(table1_bad_rows), self.table1Control.shape[0])
self.assertEqual(len(table2_good_rows)+len(table2_bad_rows), self.table2Control.shape[0])
if os.path.exists(fileName):
os.unlink(fileName)
def testConstraint(self):
"""
Test that CompoundInstanceCatalog handles constraint
properly
"""
fileName = os.path.join(self.baseDir, 'compound_constraint_test_cat.txt')
compoundCat = CompoundInstanceCatalog([Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1],
constraint='mag>20.0')
compoundCat.write_catalog(fileName)
dtype=numpy.dtype([
('id', numpy.int),
('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)
])
testData = numpy.genfromtxt(fileName, dtype=dtype)
table1_good_rows = []
table2_good_rows = []
for line in testData:
if line[0]<2000:
ix = line[0]-1000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table1Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['mag'][ix]>20.0)
elif line[0]<3000:
ix = line[0]-2000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(line[1], 2.0*self.table1Control['ra'][ix]+self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(line[2], 2.0*self.table1Control['dec'][ix]+self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(line[3], self.table1Control['mag'][ix]+self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['mag'][ix]>20.0)
else:
ix = line[0]-3000
if ix not in table2_good_rows:
table2_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix], 6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix], 6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix], 6)
self.assertTrue(self.table2Control['mag'][ix]>20.0)
table1_bad_rows = [ix for ix in range(self.table1Control.shape[0]) if ix not in table1_good_rows]
table2_bad_rows = [ix for ix in range(self.table2Control.shape[0]) if ix not in table2_good_rows]
in_bounds = [mm>20.0 for mm in self.table1Control['mag'][table1_bad_rows]]
self.assertFalse(True in in_bounds)
in_bounds = [mm>20.0 for mm in self.table2Control['mag'][table2_bad_rows]]
self.assertFalse(True in in_bounds)
self.assertTrue(len(table1_good_rows)>0)
self.assertTrue(len(table2_good_rows)>0)
self.assertTrue(len(table1_bad_rows)>0)
self.assertTrue(len(table2_bad_rows)>0)
self.assertEqual(len(table1_good_rows)+len(table1_bad_rows), self.table1Control.shape[0])
self.assertEqual(len(table2_good_rows)+len(table2_bad_rows), self.table2Control.shape[0])
if os.path.exists(fileName):
os.unlink(fileName)
def testObservationMetaDataAndConstraint(self):
"""
Test that CompoundInstanceCatalog handles ObservationMetaData
and a constraint properly
"""
fileName = os.path.join(self.baseDir,
'compound_obs_metadata_test_cat.txt')
obs = ObservationMetaData(pointingRA=180.0,
pointingDec=0.0,
boundType='box',
boundLength=(80.0, 25.0),
mjd=53850.0)
compoundCat = CompoundInstanceCatalog(
[Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1],
obs_metadata=obs,
constraint='mag>20.0')
compoundCat.write_catalog(fileName)
dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)])
testData = numpy.genfromtxt(fileName, dtype=dtype)
table1_good_rows = []
table2_good_rows = []
for line in testData:
if line[0] < 2000:
ix = line[0] - 1000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table1Control['ra'][ix],
6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix],
6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['ra'][ix] > 100.0
and self.table1Control['ra'][ix] < 260.0)
self.assertTrue(self.table1Control['dec'][ix] > -25.0
and self.table1Control['dec'][ix] < 25.0)
self.assertTrue(self.table1Control['mag'][ix] > 20.0)
elif line[0] < 3000:
ix = line[0] - 2000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(
line[1], 2.0 * self.table1Control['ra'][ix] +
self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(
line[2], 2.0 * self.table1Control['dec'][ix] +
self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['ra'][ix] > 100.0
and self.table1Control['ra'][ix] < 260.0)
self.assertTrue(self.table1Control['dec'][ix] > -25.0
and self.table1Control['dec'][ix] < 25.0)
self.assertTrue(self.table1Control['mag'][ix] > 20.0)
else:
ix = line[0] - 3000
if ix not in table2_good_rows:
table2_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix],
6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix],
6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix],
6)
self.assertTrue(self.table2Control['ra'][ix] > 100.0
and self.table2Control['ra'][ix] < 260.0)
self.assertTrue(self.table2Control['dec'][ix] > -25.0
and self.table2Control['dec'][ix] < 25.0)
self.assertTrue(self.table2Control['mag'][ix] > 20.0)
table1_bad_rows = [
ix for ix in range(self.table1Control.shape[0])
if ix not in table1_good_rows
]
table2_bad_rows = [
ix for ix in range(self.table2Control.shape[0])
if ix not in table2_good_rows
]
in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \
for rr, dd, mm in zip(self.table1Control['ra'][table1_bad_rows],
self.table1Control['dec'][table1_bad_rows],
self.table1Control['mag'][table1_bad_rows])]
self.assertFalse(True in in_bounds)
in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>20.0 \
for rr, dd, mm in zip(self.table2Control['ra'][table2_bad_rows],
self.table2Control['dec'][table2_bad_rows],
self.table2Control['mag'][table2_bad_rows])]
self.assertFalse(True in in_bounds)
self.assertTrue(len(table1_good_rows) > 0)
self.assertTrue(len(table2_good_rows) > 0)
self.assertTrue(len(table1_bad_rows) > 0)
self.assertTrue(len(table2_bad_rows) > 0)
self.assertEqual(
len(table1_good_rows) + len(table1_bad_rows),
self.table1Control.shape[0])
self.assertEqual(
len(table2_good_rows) + len(table2_bad_rows),
self.table2Control.shape[0])
if os.path.exists(fileName):
os.unlink(fileName)
def testConstraint(self):
"""
Test that CompoundInstanceCatalog handles constraint
properly
"""
fileName = os.path.join(self.baseDir,
'compound_constraint_test_cat.txt')
compoundCat = CompoundInstanceCatalog(
[Cat1, Cat2, Cat3], [table1DB1, table1DB2, table2DB1],
constraint='mag>20.0')
compoundCat.write_catalog(fileName)
dtype = numpy.dtype([('id', numpy.int), ('raObs', numpy.float),
('decObs', numpy.float),
('final_mag', numpy.float)])
testData = numpy.genfromtxt(fileName, dtype=dtype)
table1_good_rows = []
table2_good_rows = []
for line in testData:
if line[0] < 2000:
ix = line[0] - 1000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table1Control['ra'][ix],
6)
self.assertAlmostEqual(line[2], self.table1Control['dec'][ix],
6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['mag'][ix] > 20.0)
elif line[0] < 3000:
ix = line[0] - 2000
if ix not in table1_good_rows:
table1_good_rows.append(ix)
self.assertAlmostEqual(
line[1], 2.0 * self.table1Control['ra'][ix] +
self.table1Control['dra'][ix], 6)
self.assertAlmostEqual(
line[2], 2.0 * self.table1Control['dec'][ix] +
self.table1Control['ddec'][ix], 6)
self.assertAlmostEqual(
line[3], self.table1Control['mag'][ix] +
self.table1Control['dmag'][ix], 6)
self.assertTrue(self.table1Control['mag'][ix] > 20.0)
else:
ix = line[0] - 3000
if ix not in table2_good_rows:
table2_good_rows.append(ix)
self.assertAlmostEqual(line[1], self.table2Control['ra'][ix],
6)
self.assertAlmostEqual(line[2], self.table2Control['dec'][ix],
6)
self.assertAlmostEqual(line[3], self.table2Control['mag'][ix],
6)
self.assertTrue(self.table2Control['mag'][ix] > 20.0)
table1_bad_rows = [
ix for ix in range(self.table1Control.shape[0])
if ix not in table1_good_rows
]
table2_bad_rows = [
ix for ix in range(self.table2Control.shape[0])
if ix not in table2_good_rows
]
in_bounds = [
mm > 20.0 for mm in self.table1Control['mag'][table1_bad_rows]
]
self.assertFalse(True in in_bounds)
in_bounds = [
mm > 20.0 for mm in self.table2Control['mag'][table2_bad_rows]
]
self.assertFalse(True in in_bounds)
self.assertTrue(len(table1_good_rows) > 0)
self.assertTrue(len(table2_good_rows) > 0)
self.assertTrue(len(table1_bad_rows) > 0)
self.assertTrue(len(table2_bad_rows) > 0)
self.assertEqual(
len(table1_good_rows) + len(table1_bad_rows),
self.table1Control.shape[0])
self.assertEqual(
len(table2_good_rows) + len(table2_bad_rows),
self.table2Control.shape[0])
if os.path.exists(fileName):
os.unlink(fileName)
def testGalaxyCatalog(self):
"""
Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks,
and agns using both the 'old fashioned way' (one catalog at a time), and
using CompoundInstanceCatalog
"""
controlFileName = os.path.join(self.baseDir, 'gal_compound_control.txt')
testFileName = os.path.join(self.baseDir, 'gal_compound_test.txt')
if os.path.exists(controlFileName):
os.unlink(controlFileName)
if os.path.exists(testFileName):
os.unlink(testFileName)
obs = ObservationMetaData(unrefractedRA=25.0, unrefractedDec=-45.0,
boundType='circle', boundLength=0.05)
dbBulge = GalaxyBulgeObj()
dbDisk = GalaxyDiskObj()
dbAgn = GalaxyAgnObj()
catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000)
catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
# You need to reinstantiate the catalogs because the process of writing them
# above stripped galtileid from their _active_columns, which is the only way
# CompoundInstanceCatalog can know that it needs to worry about galtileid
catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
totalCat = CompoundInstanceCatalog([catBulge, catDisk, catAgn],
obs_metadata=obs,
compoundDBclass=GalaxyTileCompoundObj)
totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000)
controlFile = open(controlFileName, 'r')
control = controlFile.readlines()
controlFile.close()
testFile = open(testFileName, 'r')
test = testFile.readlines()
testFile.close()
for line in control:
self.assertTrue(line in test)
for line in test:
self.assertTrue(line in control)
if os.path.exists(controlFileName):
os.unlink(controlFileName)
if os.path.exists(testFileName):
os.unlink(testFileName)
def generatePhosimInput(mode='a', runobsHistID=None):
if mode == 'a':
filewrite = 'append'
elif mode == 'c':
filewrite = 'clobber'
opsimDB = os.path.join('.', 'kraken_1042_sqlite.db')
logfilename = 'run.log'
if os.path.isfile(logfilename):
if filewrite == 'append':
pass
elif filewrite == 'clobber':
with open('run.log', 'w') as f:
f.write('obsHistID,status,timestamp\n')
else:
print('file exists and mode uncertain')
exit()
else:
with open('run.log', 'w') as f:
f.write('obsHistID,status,time\n')
#you need to provide ObservationMetaDataGenerator with the connection
#string to an OpSim output database. This is the connection string
#to a test database that comes when you install CatSim.
generator = ObservationMetaDataGenerator(database=opsimDB, driver='sqlite')
obsHistIDList = numpy.genfromtxt('FirstSet_obsHistIDs.csv',
delimiter=',',
usecols=0)
obsMetaDataResults = []
# Change the slicing in this line for the range of visits
for obsHistID in obsHistIDList[600:700]:
if runobsHistID is not None:
obsHistID = runobsHistID
obsMetaDataResults.append(
generator.getObservationMetaData(obsHistID=obsHistID,
fieldRA=(53, 54),
fieldDec=(-29, -27),
boundLength=0.3)[0])
starObjNames = [
'msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars'
]
snmodel = SNObj()
for obs_metadata in obsMetaDataResults:
filename = "InstanceCatalogs / phosim_input_%s.txt" \
%(obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
obs_metadata.phoSimMetaData['SIM_NSNAP'] = (1, numpy.dtype(int))
obs_metadata.phoSimMetaData['SIM_VISTIME'] = (30, numpy.dtype(float))
print('Starting Visit: ',
obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
compoundStarDBList = [
MsStarObj, BhbStarObj, WdStarObj, RRLyStarObj, CepheidStarObj
]
compoundGalDBList = [GalaxyBulgeObj, GalaxyDiskObj, GalaxyAgnObj]
compoundStarICList = [
PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint,
PhoSimCatalogPoint, PhoSimCatalogPoint
]
compoundGalICList = [
PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, TwinklesCatalogZPoint
]
snphosim = PhoSimCatalogSN(db_obj=snmodel,
obs_metadata=obs_metadata,
column_outputs=['EBV'])
snphosim.writeSedFile = True
snphosim.suppressDimSN = True
snphosim.prefix = 'spectra_files/'
while True:
try:
starCat = CompoundInstanceCatalog(
compoundStarICList,
compoundStarDBList,
obs_metadata=obs_metadata,
constraint='gmag > 11.',
compoundDBclass=sprinklerCompound)
starCat.write_catalog(filename, chunk_size=10000)
galCat = CompoundInstanceCatalog(
compoundGalICList,
compoundGalDBList,
obs_metadata=obs_metadata,
# constraint='g_ab > 11.',
compoundDBclass=sprinklerCompound)
galCat.write_catalog(filename,
write_mode='a',
write_header=False,
chunk_size=10000)
snphosim.write_catalog(filename,
write_header=False,
write_mode='a',
chunk_size=10000)
if runobsHistID is not None:
print('Done doing requested obsHistID')
sys.exit()
with open(logfilename, 'a') as f:
f.write('{0:d},DONE,{1:3.6f}\n'.format(
obs_metadata.phoSimMetaData['Opsim_obshistid'][0],
time.time()))
break
except RuntimeError:
continue
print("Finished Writing Visit: ",
obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
