def testNoneMapping(self): """ Test that Nones are handled correctly in the CatalogDBObject column mappings """ db1 = dbClass1(database=self.dbName, driver='sqlite') db2 = dbClass6(database=self.dbName, driver='sqlite') colNames = ['class1_aa', 'class1_bb', 'class6_a', 'class6_b'] compoundDb = CompoundCatalogDBObject([db1, db2]) results = compoundDb.query_columns(colnames=colNames) for chunk in results: numpy.testing.assert_array_almost_equal(chunk['class1_aa'], self.controlArray['a'], decimal=6) numpy.testing.assert_array_equal(chunk['class1_bb'], self.controlArray['d']) numpy.testing.assert_array_almost_equal(chunk['class6_a'], self.controlArray['a'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class6_b'], self.controlArray['b'], decimal=6)
def testExceptions(self): """ Verify that CompoundCatalogDBObject raises an exception when you violate its API """ # test case where they are querying the same database, but different # tables db1 = dbClass1(database=self.otherDbName, driver='sqlite') db2 = dbClass2(database=self.dbName, driver='sqlite') with self.assertRaises(RuntimeError) as context: compound = CompoundCatalogDBObject([db1, db2]) self.assertTrue("['%s', '%s']" % (self.otherDbName, self.dbName) \ in context.exception.message) # test case where they are querying the same table, but different # databases db1 = dbClass4(database=self.dbName, driver='sqlite') db2 = dbClass2(database=self.dbName, driver='sqlite') with self.assertRaises(RuntimeError) as context: compound = CompoundCatalogDBObject([db1, db2]) self.assertTrue("['otherTest', 'test']" in context.exception.message) # test case where the CatalogDBObjects have the same objid db1 = dbClass4(database=self.dbName, driver='sqlite') db2 = dbClass5(database=self.dbName, driver='sqlite') with self.assertRaises(RuntimeError) as context: compound = CompoundCatalogDBObject([db1, db2]) self.assertTrue("objid class4 is duplicated" in context.exception.message) # test case where CompoundCatalogDBObject does not support the # tables being queried db1 = dbClass1(database=self.dbName, driver='sqlite') db2 = dbClass2(database=self.dbName, driver='sqlite') with self.assertRaises(RuntimeError) as context: compound = specificCompoundObj_otherTest([db1, db2]) msg = "This CompoundCatalogDBObject does not support the table 'test'" self.assertTrue(msg in context.exception.message)
def testChunks(self): """ Verify that CompoundCatalogDBObject handles chunk_size correctly """ db1 = dbClass1(database=self.dbName, driver='sqlite') db2 = dbClass2(database=self.dbName, driver='sqlite') db3 = dbClass3(database=self.dbName, driver='sqlite') dbList = [db1, db2, db3] compoundDb = CompoundCatalogDBObject(dbList) colNames = ['id', 'class1_aa', 'class1_bb', 'class2_aa', 'class2_bb', 'class3_aa', 'class3_bb', 'class3_cc'] results = compoundDb.query_columns(colnames=colNames, chunk_size=10) ct = 0 for chunk in results: ct += len(chunk['class1_aa']) rows = chunk['id'] self.assertTrue(len(rows)<=10) numpy.testing.assert_array_almost_equal(chunk['class1_aa'], self.controlArray['a'][rows], decimal=6) numpy.testing.assert_array_equal(chunk['class1_bb'], self.controlArray['d'][rows]) numpy.testing.assert_array_almost_equal(chunk['class2_aa'], 2.0*self.controlArray['b'][rows], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class2_bb'], self.controlArray['a'][rows], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class3_aa'], self.controlArray['c'][rows]-3.0, decimal=6) numpy.testing.assert_array_almost_equal(chunk['class3_bb'], self.controlArray['a'][rows], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class3_cc'], 3.0*self.controlArray['b'][rows], decimal=6) self.assertEqual(ct, 100)
def testObsMetadataAndConstraint(self): """ Test that CompoundCatalogDBObject correctly handles an ObservationMetaData and a constraint at the same time """ obs = ObservationMetaData(unrefractedRA = 180.0, unrefractedDec = 0.0, boundType = 'box', boundLength = (80.0, 25.0)) db1 = testStarDB1(database=self.dbName, driver='sqlite') db2 = testStarDB2(database=self.dbName, driver='sqlite') compoundDb = CompoundCatalogDBObject([db1, db2]) colnames = ['testStar1_id', 'testStar1_raJ2000', 'testStar1_decJ2000', 'testStar1_magMod', 'testStar2_raJ2000', 'testStar2_decJ2000', 'testStar2_magMod'] results = compoundDb.query_columns(colnames=colnames, obs_metadata=obs, constraint='mag>15.0') good_rows = [] for chunk in results: for line in chunk: ix = line['id'] good_rows.append(ix) self.assertAlmostEqual(line['testStar1_raJ2000'], self.controlArray['ra'][ix], 10) self.assertAlmostEqual(line['testStar1_decJ2000'], self.controlArray['dec'][ix], 10) self.assertAlmostEqual(line['testStar1_magMod'], self.controlArray['mag'][ix], 10) self.assertAlmostEqual(line['testStar2_raJ2000'], 2.0*self.controlArray['ra'][ix], 10) self.assertAlmostEqual(line['testStar2_decJ2000'], 2.0*self.controlArray['dec'][ix], 10) self.assertAlmostEqual(line['testStar2_magMod'], 2.0*self.controlArray['mag'][ix], 10) self.assertTrue(self.controlArray['ra'][ix]>100.0) self.assertTrue(self.controlArray['ra'][ix]<260.0) self.assertTrue(self.controlArray['dec'][ix]>-25.0) self.assertTrue(self.controlArray['dec'][ix]<25.0) self.assertTrue(self.controlArray['mag'][ix]>15.0) bad_rows = [ix for ix in range(self.controlArray.shape[0]) if ix not in good_rows] in_bounds = [rr>100.0 and rr<260.0 and dd>-25.0 and dd<25.0 and mm>150.0 \ for (rr, dd, mm) in \ zip(self.controlArray['ra'][bad_rows], self.controlArray['dec'][bad_rows], \ self.controlArray['mag'][bad_rows])] self.assertFalse(True in in_bounds) self.assertTrue(len(good_rows)>0) self.assertTrue(len(bad_rows)>0) self.assertEqual(len(good_rows)+len(bad_rows), self.controlArray.shape[0])
def testContraint(self): """ Test that CompoundCatalogDBObject runs correctly with a constraint """ class testDbClass24(testStarDB1): database = self.dbName driver = 'sqlite' class testDbClass25(testStarDB2): database = self.dbName driver = 'sqlite' db1 = testDbClass24 db2 = testDbClass25 compoundDb = CompoundCatalogDBObject([db1, db2]) colnames = ['%s_id' % db1.objid, '%s_raJ2000' % db1.objid, '%s_decJ2000' % db1.objid, '%s_magMod' % db1.objid, '%s_raJ2000' % db2.objid, '%s_decJ2000' % db2.objid, '%s_magMod' % db2.objid] results = compoundDb.query_columns(colnames=colnames, constraint='mag<11.0') good_rows = [] for chunk in results: for line in chunk: ix = line['id'] good_rows.append(ix) self.assertAlmostEqual(line['%s_raJ2000' % db1.objid], self.controlArray['ra'][ix], 10) self.assertAlmostEqual(line['%s_decJ2000' % db1.objid], self.controlArray['dec'][ix], 10) self.assertAlmostEqual(line['%s_magMod' % db1.objid], self.controlArray['mag'][ix], 10) self.assertAlmostEqual(line['%s_raJ2000' % db2.objid], 2.0*self.controlArray['ra'][ix], 10) self.assertAlmostEqual(line['%s_decJ2000' % db2.objid], 2.0*self.controlArray['dec'][ix], 10) self.assertAlmostEqual(line['%s_magMod' % db2.objid], 2.0*self.controlArray['mag'][ix], 10) self.assertTrue(self.controlArray['mag'][ix]<11.0) bad_rows = [ix for ix in range(self.controlArray.shape[0]) if ix not in good_rows] in_bounds = [mm<11.0 for mm in self.controlArray['mag'][bad_rows]] self.assertFalse(True in in_bounds) self.assertTrue(len(good_rows)>0) self.assertTrue(len(bad_rows)>0) self.assertEqual(len(good_rows)+len(bad_rows), self.controlArray.shape[0])
def testCompoundCatalogDBObject(self): """ Verify that CompoundCatalogDBObject returns the expected columns. """ db1 = dbClass1(database=self.dbName, driver='sqlite') db2 = dbClass2(database=self.dbName, driver='sqlite') db3 = dbClass3(database=self.dbName, driver='sqlite') dbList = [db1, db2, db3] compoundDb = CompoundCatalogDBObject(dbList) colNames = ['class1_aa', 'class1_bb', 'class2_aa', 'class2_bb', 'class3_aa', 'class3_bb', 'class3_cc'] results = compoundDb.query_columns(colnames=colNames) for chunk in results: numpy.testing.assert_array_almost_equal(chunk['class1_aa'], self.controlArray['a'], decimal=6) numpy.testing.assert_array_equal(chunk['class1_bb'], self.controlArray['d']) numpy.testing.assert_array_almost_equal(chunk['class2_aa'], 2.0*self.controlArray['b'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class2_bb'], self.controlArray['a'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class3_aa'], self.controlArray['c']-3.0, decimal=6) numpy.testing.assert_array_almost_equal(chunk['class3_bb'], self.controlArray['a'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['class3_cc'], 3.0*self.controlArray['b'], decimal=6)
def testNoneMapping(self): """ Test that Nones are handled correctly in the CatalogDBObject column mappings """ class testDbClass20(dbClass1): database = self.dbName driver = 'sqlite' class testDbClass21(dbClass6): database = self.dbName driver = 'sqlite' db1 = testDbClass20 db2 = testDbClass21 colNames = ['%s_aa' % db1.objid, '%s_bb' % db1.objid, '%s_a' % db2.objid, '%s_b' % db2.objid] compoundDb = CompoundCatalogDBObject([db1, db2]) results = compoundDb.query_columns(colnames=colNames) for chunk in results: numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db1.objid], self.controlArray['a'], decimal=6) numpy.testing.assert_array_equal(chunk['%s_bb' % db1.objid], self.controlArray['d']) numpy.testing.assert_array_almost_equal(chunk['%s_a' % db2.objid], self.controlArray['a'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_b' % db2.objid], self.controlArray['b'], decimal=6)
def testChunks(self): """ Verify that CompoundCatalogDBObject handles chunk_size correctly """ class testDbClass17(dbClass1): database = self.dbName driver = 'sqlite' class testDbClass18(dbClass2): database = self.dbName driver = 'sqlite' class testDbClass19(dbClass3): database = self.dbName driver = 'sqlite' db1 = testDbClass17 db2 = testDbClass18 db3 = testDbClass19 dbList = [db1, db2, db3] compoundDb = CompoundCatalogDBObject(dbList) colNames = ['id', '%s_aa' % db1.objid, '%s_bb' % db1.objid, '%s_aa' % db2.objid, '%s_bb' % db2.objid, '%s_aa' % db3.objid, '%s_bb' % db3.objid, '%s_cc' % db3.objid] results = compoundDb.query_columns(colnames=colNames, chunk_size=10) ct = 0 for chunk in results: ct += len(chunk['%s_aa' % db1.objid]) rows = chunk['id'] self.assertTrue(len(rows)<=10) numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db1.objid], self.controlArray['a'][rows], decimal=6) numpy.testing.assert_array_equal(chunk['%s_bb' % db1.objid], self.controlArray['d'][rows]) numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db2.objid], 2.0*self.controlArray['b'][rows], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db2.objid], self.controlArray['a'][rows], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db3.objid], self.controlArray['c'][rows]-3.0, decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db3.objid], self.controlArray['a'][rows], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_cc' % db3.objid], 3.0*self.controlArray['b'][rows], decimal=6) self.assertEqual(ct, 100)
def testCompoundCatalogDBObject(self): """ Verify that CompoundCatalogDBObject returns the expected columns. """ class testDbClass9(dbClass1): database = self.dbName driver = 'sqlite' class testDbClass10(dbClass2): database = self.dbName driver = 'sqlite' class testDbClass11(dbClass3): database = self.dbName driver = 'sqlite' db1 = testDbClass9 db2 = testDbClass10 db3 = testDbClass11 dbList = [db1, db2, db3] compoundDb = CompoundCatalogDBObject(dbList) colNames = ['%s_aa' % db1.objid, '%s_bb' % db1.objid, '%s_aa' % db2.objid, '%s_bb' % db2.objid, '%s_aa' % db3.objid, '%s_bb' % db3.objid, '%s_cc' % db3.objid] results = compoundDb.query_columns(colnames=colNames) for chunk in results: numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db1.objid], self.controlArray['a'], decimal=6) numpy.testing.assert_array_equal(chunk['%s_bb' % db1.objid], self.controlArray['d']) numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db2.objid], 2.0*self.controlArray['b'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db2.objid], self.controlArray['a'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_aa' % db3.objid], self.controlArray['c']-3.0, decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_bb' % db3.objid], self.controlArray['a'], decimal=6) numpy.testing.assert_array_almost_equal(chunk['%s_cc' % db3.objid], 3.0*self.controlArray['b'], decimal=6)
def write_catalog(self, filename, chunk_size=None, write_header=True, write_mode='w'): """ Write the stored list of InstanceCatalogs to a single ASCII output catalog. @param [in] filename is the name of the file to be written @param [in] chunk_size is an optional parameter telling the CompoundInstanceCatalog to query the database in manageable chunks (in case returning the whole catalog takes too much memory) @param [in] write_header a boolean specifying whether or not to add a header to the output catalog (Note: only one header will be written; there will not be a header for each InstanceCatalog in the CompoundInstanceCatalog; default True) @param [in] write_mode is 'w' if you want to overwrite the output file or 'a' if you want to append to an existing output file (default: 'w') """ instantiated_ic_list = [None]*len(self._ic_list) # first, loop over all of the InstanceCatalog and CatalogDBObject classes, pre-processing # them (i.e. verifying that they have access to all of the columns they need) for ix, (icClass, dboClass) in enumerate(zip(self._ic_list, self._dbo_list)): best_connection = self.find_a_connection(dboClass) if best_connection is None: dbo = dboClass() self._active_connections.append(dbo.connection) else: dbo = dboClass(connection=best_connection) ic = icClass(dbo, obs_metadata=self._obs_metadata) ic._write_pre_process() instantiated_ic_list[ix] = ic for row in self._dbObjectGroupList: if len(row)==1: ic = instantiated_ic_list[row[0]] ic._query_and_write(filename, chunk_size=chunk_size, write_header=write_header, write_mode=write_mode, obs_metadata=self._obs_metadata, constraint=self._constraint) write_mode = 'a' write_header = False default_compound_dbo = None if self._compoundDBclass is not None: if not hasattr(self._compoundDBclass, '__getitem__'): default_compound_dbo = CompoundCatalogDBObject else: for dbo in self._compoundDBclass: if dbo._table_restriction is None: default_compound_dbo = dbo break if default_compound_dbo is None: default_compound_dbo is CompoundCatalogDBObject for row in self._dbObjectGroupList: if len(row)>1: dbObjClassList = [self._dbo_list[ix] for ix in row] catList = [instantiated_ic_list[ix] for ix in row] # if a connection is already open to the database, use # it rather than opening a new connection best_connection = self.find_a_connection(dbObjClassList[0]) if self._compoundDBclass is None: compound_dbo = CompoundCatalogDBObject(dbObjClassList, connection=best_connection) elif not hasattr(self._compoundDBclass, '__getitem__'): # if self._compoundDBclass is not a list try: compound_dbo = self._compoundDBclass(dbObjClassList) except: compound_dbo = default_compound_dbo(dbObjClassList) else: compound_dbo = None for candidate in self._compoundDBclass: use_it = True if False in [candidate._table_restriction is not None \ and dbo.tableid in candidate._table_restriction \ for dbo in dbObjClassList]: use_it = False if use_it: compound_dbo = candidate(dbObjClassList) break if compound_dbo is None: compound_dbo = default_compound_dbo(dbObjClassList) self._write_compound(catList, compound_dbo, filename, chunk_size=chunk_size, write_header=write_header, write_mode=write_mode) write_mode = 'a' write_header = False