def testObsMetaData(self):
"""
Test that CompoundCatalogDBObject can handle ObservationMetaData
properly
"""
obs = ObservationMetaData(pointingRA = 180.0,
pointingDec = 0.0,
boundType = 'box',
boundLength = (80.0, 25.0),
mjd=53580.0)
class testDbClass22(testStarDB1):
database = self.dbName
driver = 'sqlite'
class testDbClass23(testStarDB2):
database = self.dbName
driver = 'sqlite'
db1 = testDbClass22
db2 = testDbClass23
compoundDb = CompoundCatalogDBObject([db1, db2])
colnames = ['%s_id' % db1.objid,
'%s_raJ2000' % db1.objid, '%s_decJ2000' % db1.objid,
'%s_magMod' % db1.objid,
'%s_raJ2000' % db2.objid, '%s_decJ2000' % db2.objid,
'%s_magMod' % db2.objid]
results = compoundDb.query_columns(colnames=colnames,
obs_metadata=obs)
good_rows = []
for chunk in results:
for line in chunk:
ix = line['id']
good_rows.append(ix)
self.assertAlmostEqual(line['%s_raJ2000' % db1.objid], self.controlArray['ra'][ix], 10)
self.assertAlmostEqual(line['%s_decJ2000' % db1.objid], self.controlArray['dec'][ix], 10)
self.assertAlmostEqual(line['%s_magMod' % db1.objid], self.controlArray['mag'][ix], 10)
self.assertAlmostEqual(line['%s_raJ2000' % db2.objid], 2.0*self.controlArray['ra'][ix], 10)
self.assertAlmostEqual(line['%s_decJ2000' % db2.objid], 2.0*self.controlArray['dec'][ix], 10)
self.assertAlmostEqual(line['%s_magMod' % db2.objid], 2.0*self.controlArray['mag'][ix], 10)
self.assertTrue(self.controlArray['ra'][ix]>100.0)
self.assertTrue(self.controlArray['ra'][ix]<260.0)
self.assertTrue(self.controlArray['dec'][ix]>-25.0)
self.assertTrue(self.controlArray['dec'][ix]<25.0)
bad_rows = [ix for ix in range(self.controlArray.shape[0]) if ix not in good_rows]
in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 \
for (rr, dd) in \
zip(self.controlArray['ra'][bad_rows], self.controlArray['dec'][bad_rows])]
self.assertFalse(True in in_bounds)
self.assertTrue(len(good_rows)>0)
self.assertTrue(len(bad_rows)>0)
def testNoneMapping(self):
"""
Test that Nones are handled correctly in the CatalogDBObject
column mappings
"""
db1 = dbClass1(database=self.dbName, driver='sqlite')
db2 = dbClass6(database=self.dbName, driver='sqlite')
colNames = ['class1_aa', 'class1_bb', 'class6_a', 'class6_b']
compoundDb = CompoundCatalogDBObject([db1, db2])
results = compoundDb.query_columns(colnames=colNames)
for chunk in results:
numpy.testing.assert_array_almost_equal(chunk['class1_aa'],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_equal(chunk['class1_bb'],
self.controlArray['d'])
numpy.testing.assert_array_almost_equal(chunk['class6_a'],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class6_b'],
self.controlArray['b'],
decimal=6)
def testChunks(self):
"""
Verify that CompoundCatalogDBObject handles chunk_size correctly
"""
db1 = dbClass1(database=self.dbName, driver='sqlite')
db2 = dbClass2(database=self.dbName, driver='sqlite')
db3 = dbClass3(database=self.dbName, driver='sqlite')
dbList = [db1, db2, db3]
compoundDb = CompoundCatalogDBObject(dbList)
colNames = ['id',
'class1_aa', 'class1_bb',
'class2_aa', 'class2_bb',
'class3_aa', 'class3_bb', 'class3_cc']
results = compoundDb.query_columns(colnames=colNames, chunk_size=10)
ct = 0
for chunk in results:
ct += len(chunk['class1_aa'])
rows = chunk['id']
self.assertTrue(len(rows)<=10)
numpy.testing.assert_array_almost_equal(chunk['class1_aa'],
self.controlArray['a'][rows],
decimal=6)
numpy.testing.assert_array_equal(chunk['class1_bb'],
self.controlArray['d'][rows])
numpy.testing.assert_array_almost_equal(chunk['class2_aa'],
2.0*self.controlArray['b'][rows],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class2_bb'],
self.controlArray['a'][rows],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class3_aa'],
self.controlArray['c'][rows]-3.0,
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class3_bb'],
self.controlArray['a'][rows],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class3_cc'],
3.0*self.controlArray['b'][rows],
decimal=6)
self.assertEqual(ct, 100)
def testObsMetadataAndConstraint(self):
"""
Test that CompoundCatalogDBObject correctly handles an ObservationMetaData
and a constraint at the same time
"""
obs = ObservationMetaData(unrefractedRA = 180.0,
unrefractedDec = 0.0,
boundType = 'box',
boundLength = (80.0, 25.0))
db1 = testStarDB1(database=self.dbName, driver='sqlite')
db2 = testStarDB2(database=self.dbName, driver='sqlite')
compoundDb = CompoundCatalogDBObject([db1, db2])
colnames = ['testStar1_id',
'testStar1_raJ2000', 'testStar1_decJ2000', 'testStar1_magMod',
'testStar2_raJ2000', 'testStar2_decJ2000', 'testStar2_magMod']
results = compoundDb.query_columns(colnames=colnames,
obs_metadata=obs,
constraint='mag>15.0')
good_rows = []
for chunk in results:
for line in chunk:
ix = line['id']
good_rows.append(ix)
self.assertAlmostEqual(line['testStar1_raJ2000'], self.controlArray['ra'][ix], 10)
self.assertAlmostEqual(line['testStar1_decJ2000'], self.controlArray['dec'][ix], 10)
self.assertAlmostEqual(line['testStar1_magMod'], self.controlArray['mag'][ix], 10)
self.assertAlmostEqual(line['testStar2_raJ2000'], 2.0*self.controlArray['ra'][ix], 10)
self.assertAlmostEqual(line['testStar2_decJ2000'], 2.0*self.controlArray['dec'][ix], 10)
self.assertAlmostEqual(line['testStar2_magMod'], 2.0*self.controlArray['mag'][ix], 10)
self.assertTrue(self.controlArray['ra'][ix]>100.0)
self.assertTrue(self.controlArray['ra'][ix]<260.0)
self.assertTrue(self.controlArray['dec'][ix]>-25.0)
self.assertTrue(self.controlArray['dec'][ix]<25.0)
self.assertTrue(self.controlArray['mag'][ix]>15.0)
bad_rows = [ix for ix in range(self.controlArray.shape[0]) if ix not in good_rows]
in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>150.0 \
for (rr, dd, mm) in \
zip(self.controlArray['ra'][bad_rows], self.controlArray['dec'][bad_rows], \
self.controlArray['mag'][bad_rows])]
self.assertFalse(True in in_bounds)
self.assertTrue(len(good_rows)>0)
self.assertTrue(len(bad_rows)>0)
self.assertEqual(len(good_rows)+len(bad_rows), self.controlArray.shape[0])
def testContraint(self):
"""
Test that CompoundCatalogDBObject runs correctly with a constraint
"""
class testDbClass24(testStarDB1):
database = self.dbName
driver = 'sqlite'
class testDbClass25(testStarDB2):
database = self.dbName
driver = 'sqlite'
db1 = testDbClass24
db2 = testDbClass25
compoundDb = CompoundCatalogDBObject([db1, db2])
colnames = ['%s_id' % db1.objid,
'%s_raJ2000' % db1.objid, '%s_decJ2000' % db1.objid,
'%s_magMod' % db1.objid,
'%s_raJ2000' % db2.objid, '%s_decJ2000' % db2.objid,
'%s_magMod' % db2.objid]
results = compoundDb.query_columns(colnames=colnames,
constraint='mag<11.0')
good_rows = []
for chunk in results:
for line in chunk:
ix = line['id']
good_rows.append(ix)
self.assertAlmostEqual(line['%s_raJ2000' % db1.objid], self.controlArray['ra'][ix], 10)
self.assertAlmostEqual(line['%s_decJ2000' % db1.objid], self.controlArray['dec'][ix], 10)
self.assertAlmostEqual(line['%s_magMod' % db1.objid], self.controlArray['mag'][ix], 10)
self.assertAlmostEqual(line['%s_raJ2000' % db2.objid], 2.0*self.controlArray['ra'][ix], 10)
self.assertAlmostEqual(line['%s_decJ2000' % db2.objid], 2.0*self.controlArray['dec'][ix], 10)
self.assertAlmostEqual(line['%s_magMod' % db2.objid], 2.0*self.controlArray['mag'][ix], 10)
self.assertTrue(self.controlArray['mag'][ix]<11.0)
bad_rows = [ix for ix in range(self.controlArray.shape[0]) if ix not in good_rows]
in_bounds = [mm<11.0 for mm in self.controlArray['mag'][bad_rows]]
self.assertFalse(True in in_bounds)
self.assertTrue(len(good_rows)>0)
self.assertTrue(len(bad_rows)>0)
self.assertEqual(len(good_rows)+len(bad_rows), self.controlArray.shape[0])
def testCompoundCatalogDBObject(self):
"""
Verify that CompoundCatalogDBObject returns the expected
columns.
"""
db1 = dbClass1(database=self.dbName, driver='sqlite')
db2 = dbClass2(database=self.dbName, driver='sqlite')
db3 = dbClass3(database=self.dbName, driver='sqlite')
dbList = [db1, db2, db3]
compoundDb = CompoundCatalogDBObject(dbList)
colNames = ['class1_aa', 'class1_bb',
'class2_aa', 'class2_bb',
'class3_aa', 'class3_bb', 'class3_cc']
results = compoundDb.query_columns(colnames=colNames)
for chunk in results:
numpy.testing.assert_array_almost_equal(chunk['class1_aa'],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_equal(chunk['class1_bb'],
self.controlArray['d'])
numpy.testing.assert_array_almost_equal(chunk['class2_aa'],
2.0*self.controlArray['b'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class2_bb'],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class3_aa'],
self.controlArray['c']-3.0,
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class3_bb'],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['class3_cc'],
3.0*self.controlArray['b'],
decimal=6)
def testExceptions(self):
"""
Verify that CompoundCatalogDBObject raises an exception
when you violate its API
"""
# test case where they are querying the same database, but different
# tables
db1 = dbClass1(database=self.otherDbName, driver='sqlite')
db2 = dbClass2(database=self.dbName, driver='sqlite')
with self.assertRaises(RuntimeError) as context:
compound = CompoundCatalogDBObject([db1, db2])
self.assertTrue("['%s', '%s']" % (self.otherDbName, self.dbName) \
in context.exception.message)
# test case where they are querying the same table, but different
# databases
db1 = dbClass4(database=self.dbName, driver='sqlite')
db2 = dbClass2(database=self.dbName, driver='sqlite')
with self.assertRaises(RuntimeError) as context:
compound = CompoundCatalogDBObject([db1, db2])
self.assertTrue("['otherTest', 'test']" in context.exception.message)
# test case where the CatalogDBObjects have the same objid
db1 = dbClass4(database=self.dbName, driver='sqlite')
db2 = dbClass5(database=self.dbName, driver='sqlite')
with self.assertRaises(RuntimeError) as context:
compound = CompoundCatalogDBObject([db1, db2])
self.assertTrue("objid class4 is duplicated" in context.exception.message)
# test case where CompoundCatalogDBObject does not support the
# tables being queried
db1 = dbClass1(database=self.dbName, driver='sqlite')
db2 = dbClass2(database=self.dbName, driver='sqlite')
with self.assertRaises(RuntimeError) as context:
compound = specificCompoundObj_otherTest([db1, db2])
msg = "This CompoundCatalogDBObject does not support the table 'test'"
self.assertTrue(msg in context.exception.message)
def testNoneMapping(self):
"""
Test that Nones are handled correctly in the CatalogDBObject
column mappings
"""
class testDbClass20(dbClass1):
database = self.dbName
driver = 'sqlite'
class testDbClass21(dbClass6):
database = self.dbName
driver = 'sqlite'
db1 = testDbClass20
db2 = testDbClass21
colNames = ['%s_aa' % db1.objid, '%s_bb' % db1.objid,
'%s_a' % db2.objid, '%s_b' % db2.objid]
compoundDb = CompoundCatalogDBObject([db1, db2])
results = compoundDb.query_columns(colnames=colNames)
for chunk in results:
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db1.objid],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_equal(chunk['%s_bb' % db1.objid],
self.controlArray['d'])
numpy.testing.assert_array_almost_equal(chunk['%s_a' % db2.objid],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_b' % db2.objid],
self.controlArray['b'],
decimal=6)
def testChunks(self):
"""
Verify that CompoundCatalogDBObject handles chunk_size correctly
"""
class testDbClass17(dbClass1):
database = self.dbName
driver = 'sqlite'
class testDbClass18(dbClass2):
database = self.dbName
driver = 'sqlite'
class testDbClass19(dbClass3):
database = self.dbName
driver = 'sqlite'
db1 = testDbClass17
db2 = testDbClass18
db3 = testDbClass19
dbList = [db1, db2, db3]
compoundDb = CompoundCatalogDBObject(dbList)
colNames = ['id',
'%s_aa' % db1.objid, '%s_bb' % db1.objid,
'%s_aa' % db2.objid, '%s_bb' % db2.objid,
'%s_aa' % db3.objid, '%s_bb' % db3.objid,
'%s_cc' % db3.objid]
results = compoundDb.query_columns(colnames=colNames, chunk_size=10)
ct = 0
for chunk in results:
ct += len(chunk['%s_aa' % db1.objid])
rows = chunk['id']
self.assertTrue(len(rows)<=10)
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db1.objid],
self.controlArray['a'][rows],
decimal=6)
numpy.testing.assert_array_equal(chunk['%s_bb' % db1.objid],
self.controlArray['d'][rows])
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db2.objid],
2.0*self.controlArray['b'][rows],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db2.objid],
self.controlArray['a'][rows],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db3.objid],
self.controlArray['c'][rows]-3.0,
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db3.objid],
self.controlArray['a'][rows],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_cc' % db3.objid],
3.0*self.controlArray['b'][rows],
decimal=6)
self.assertEqual(ct, 100)
def testCompoundCatalogDBObject(self):
"""
Verify that CompoundCatalogDBObject returns the expected
columns.
"""
class testDbClass9(dbClass1):
database = self.dbName
driver = 'sqlite'
class testDbClass10(dbClass2):
database = self.dbName
driver = 'sqlite'
class testDbClass11(dbClass3):
database = self.dbName
driver = 'sqlite'
db1 = testDbClass9
db2 = testDbClass10
db3 = testDbClass11
dbList = [db1, db2, db3]
compoundDb = CompoundCatalogDBObject(dbList)
colNames = ['%s_aa' % db1.objid, '%s_bb' % db1.objid,
'%s_aa' % db2.objid, '%s_bb' % db2.objid,
'%s_aa' % db3.objid, '%s_bb' % db3.objid,
'%s_cc' % db3.objid]
results = compoundDb.query_columns(colnames=colNames)
for chunk in results:
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db1.objid],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_equal(chunk['%s_bb' % db1.objid],
self.controlArray['d'])
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db2.objid],
2.0*self.controlArray['b'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db2.objid],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db3.objid],
self.controlArray['c']-3.0,
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db3.objid],
self.controlArray['a'],
decimal=6)
numpy.testing.assert_array_almost_equal(chunk['%s_cc' % db3.objid],
3.0*self.controlArray['b'],
decimal=6)
def write_catalog(self, filename, chunk_size=None, write_header=True, write_mode='w'):
"""
Write the stored list of InstanceCatalogs to a single ASCII output catalog.
@param [in] filename is the name of the file to be written
@param [in] chunk_size is an optional parameter telling the CompoundInstanceCatalog
to query the database in manageable chunks (in case returning the whole catalog
takes too much memory)
@param [in] write_header a boolean specifying whether or not to add a header
to the output catalog (Note: only one header will be written; there will not be
a header for each InstanceCatalog in the CompoundInstanceCatalog; default True)
@param [in] write_mode is 'w' if you want to overwrite the output file or
'a' if you want to append to an existing output file (default: 'w')
"""
instantiated_ic_list = [None]*len(self._ic_list)
# first, loop over all of the InstanceCatalog and CatalogDBObject classes, pre-processing
# them (i.e. verifying that they have access to all of the columns they need)
for ix, (icClass, dboClass) in enumerate(zip(self._ic_list, self._dbo_list)):
best_connection = self.find_a_connection(dboClass)
if best_connection is None:
dbo = dboClass()
self._active_connections.append(dbo.connection)
else:
dbo = dboClass(connection=best_connection)
ic = icClass(dbo, obs_metadata=self._obs_metadata)
ic._write_pre_process()
instantiated_ic_list[ix] = ic
for row in self._dbObjectGroupList:
if len(row)==1:
ic = instantiated_ic_list[row[0]]
ic._query_and_write(filename, chunk_size=chunk_size,
write_header=write_header, write_mode=write_mode,
obs_metadata=self._obs_metadata,
constraint=self._constraint)
write_mode = 'a'
write_header = False
default_compound_dbo = None
if self._compoundDBclass is not None:
if not hasattr(self._compoundDBclass, '__getitem__'):
default_compound_dbo = CompoundCatalogDBObject
else:
for dbo in self._compoundDBclass:
if dbo._table_restriction is None:
default_compound_dbo = dbo
break
if default_compound_dbo is None:
default_compound_dbo is CompoundCatalogDBObject
for row in self._dbObjectGroupList:
if len(row)>1:
dbObjClassList = [self._dbo_list[ix] for ix in row]
catList = [instantiated_ic_list[ix] for ix in row]
# if a connection is already open to the database, use
# it rather than opening a new connection
best_connection = self.find_a_connection(dbObjClassList[0])
if self._compoundDBclass is None:
compound_dbo = CompoundCatalogDBObject(dbObjClassList, connection=best_connection)
elif not hasattr(self._compoundDBclass, '__getitem__'):
# if self._compoundDBclass is not a list
try:
compound_dbo = self._compoundDBclass(dbObjClassList)
except:
compound_dbo = default_compound_dbo(dbObjClassList)
else:
compound_dbo = None
for candidate in self._compoundDBclass:
use_it = True
if False in [candidate._table_restriction is not None \
and dbo.tableid in candidate._table_restriction \
for dbo in dbObjClassList]:
use_it = False
if use_it:
compound_dbo = candidate(dbObjClassList)
break
if compound_dbo is None:
compound_dbo = default_compound_dbo(dbObjClassList)
self._write_compound(catList, compound_dbo, filename,
chunk_size=chunk_size, write_header=write_header,
write_mode=write_mode)
write_mode = 'a'
write_header = False
