def test_suite(): suite = unittest.TestSuite() if db.version() >= (4,5) : suite.addTest(unittest.makeSuite(DBTxn_distributed)) suite.addTest(unittest.makeSuite(DBTxn_distributedSYNC)) if db.version() >= (4,6) : suite.addTest(unittest.makeSuite(DBTxn_distributed_must_open_db)) suite.addTest(unittest.makeSuite(DBTxn_distributedSYNC_must_open_db)) return suite
def test07_DB_set_flags_persists(self): if db.version() < (4,2): # The get_flags API required for this to work is only available # in Berkeley DB >= 4.2 return try: db1 = db.DB() db1.set_flags(db.DB_DUPSORT) db1.open(self.filename, db.DB_HASH, db.DB_CREATE) db1['a'] = 'eh' db1['a'] = 'A' self.assertEqual([('a', 'A')], db1.items()) db1.put('a', 'Aa') self.assertEqual([('a', 'A'), ('a', 'Aa')], db1.items()) db1.close() db1 = db.DB() # no set_flags call, we're testing that it reads and obeys # the flags on open. db1.open(self.filename, db.DB_HASH) self.assertEqual([('a', 'A'), ('a', 'Aa')], db1.items()) # if it read the flags right this will replace all values # for key 'a' instead of adding a new one. (as a dict should) db1['a'] = 'new A' self.assertEqual([('a', 'new A')], db1.items()) finally: db1.close() test_support.unlink(self.filename)
def test13_associate_in_transaction(self): if verbose: print '\n', '-=' * 30 print "Running %s.test13_associateAutoCommit..." % \ self.__class__.__name__ txn = self.env.txn_begin() try: self.createDB(txn=txn) self.secDB = db.DB(self.env) self.secDB.set_flags(db.DB_DUP) self.secDB.set_get_returns_none(2) self.secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, txn=txn) if db.version() >= (4,1): self.getDB().associate(self.secDB, self.getGenre, txn=txn) else: self.getDB().associate(self.secDB, self.getGenre) self.addDataToDB(self.getDB(), txn=txn) except: txn.abort() raise self.txn_finish_test(self.secDB, txn=txn)
def test06_Transactions(self): d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test06_Transactions..." % self.__class__.__name__ self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) self.assertEqual( d.get('new rec', txn=self.txn), 'this is a new record') self.txn.abort() self.assertEqual(d.get('new rec'), None) self.txn = self.env.txn_begin() self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) self.assertEqual( d.get('new rec', txn=self.txn), 'this is a new record') self.txn.commit() self.assertEqual(d.get('new rec'), 'this is a new record') self.txn = self.env.txn_begin() c = d.cursor(self.txn) rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec rec = c.next() self.assertEqual(count, self._numKeys + 1) c.close() # Cursors *MUST* be closed before commit! self.txn.commit() # flush pending updates try: self.env.txn_checkpoint(0, 0, 0) except db.DBIncompleteError: pass statDict = self.env.log_stat(0) self.assertTrue('magic' in statDict) self.assertTrue('version' in statDict) self.assertTrue('cur_file' in statDict) self.assertTrue('region_nowait' in statDict) # must have at least one log file present: logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG) self.assertNotEqual(logs, None) for log in logs: if verbose: print 'log file: ' + log if db.version() >= (4, 2): logs = self.env.log_archive(db.DB_ARCH_REMOVE) self.assertTrue(not logs) self.txn = self.env.txn_begin()
def test00_version(self): info = db.version() if verbose: print "\n", "-=" * 20 print "bsddb.db.version(): %s" % (info,) print db.DB_VERSION_STRING print "-=" * 20 self.assertEqual(info, (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, db.DB_VERSION_PATCH))
def test00_version(self): info = db.version() if verbose: print '\n', '-=' * 20 print 'bsddb.db.version(): %s' % (info,) print db.DB_VERSION_STRING print '-=' * 20 self.assertEqual(info, (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, db.DB_VERSION_PATCH))
def createDB(self, txn=None): self.cur = None self.secDB = None self.primary = db.DB(self.env) self.primary.set_get_returns_none(2) if db.version() >= (4, 1): self.primary.open(self.filename, "primary", self.dbtype, db.DB_CREATE | db.DB_THREAD | self.dbFlags, txn=txn) else: self.primary.open(self.filename, "primary", self.dbtype, db.DB_CREATE | db.DB_THREAD | self.dbFlags)
def test_lk_detect(self) : flags= [db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS, db.DB_LOCK_MINLOCKS, db.DB_LOCK_MINWRITE, db.DB_LOCK_OLDEST, db.DB_LOCK_RANDOM, db.DB_LOCK_YOUNGEST] if db.version() >= (4, 3) : flags.append(db.DB_LOCK_MAXWRITE) for i in flags : self.env.set_lk_detect(i) self.assertEqual(i, self.env.get_lk_detect())
def test04_lock_timeout2(self): self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT) self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT) self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT) self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT) def deadlock_detection() : while not deadlock_detection.end : deadlock_detection.count = \ self.env.lock_detect(db.DB_LOCK_EXPIRE) if deadlock_detection.count : while not deadlock_detection.end : pass break time.sleep(0.01) deadlock_detection.end=False deadlock_detection.count=0 t=Thread(target=deadlock_detection) import sys if sys.version_info[0] < 3 : t.setDaemon(True) else : t.daemon = True t.start() self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT) anID = self.env.lock_id() anID2 = self.env.lock_id() self.assertNotEqual(anID, anID2) lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE) start_time=time.time() self.assertRaises(db.DBLockNotGrantedError, self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ) end_time=time.time() deadlock_detection.end=True # Floating point rounding if sys.platform == 'win32': # bpo-30850: On Windows, tolerate 50 ms whereas 100 ms is expected. # The lock sometimes times out after only 58 ms. Windows clocks # have a bad resolution and bad accuracy. min_dt = 0.050 else: min_dt = 0.0999 self.assertGreaterEqual(end_time-start_time, min_dt) self.env.lock_put(lock) t.join() self.env.lock_id_free(anID) self.env.lock_id_free(anID2) if db.version() >= (4,6): self.assertGreater(deadlock_detection.count, 0)
def test_suite(): suite = unittest.TestSuite() if db.version() >= (4, 6): dbenv = db.DBEnv() try: dbenv.repmgr_get_ack_policy() ReplicationManager_available = True except: ReplicationManager_available = False dbenv.close() del dbenv if ReplicationManager_available: suite.addTest(unittest.makeSuite(DBReplicationManager)) if have_threads: suite.addTest(unittest.makeSuite(DBBaseReplication)) return suite
def setUp(self): if self.useEnv: self.homeDir=get_new_environment_path() try: self.env = db.DBEnv() self.env.set_lg_max(1024*1024) self.env.set_tx_max(30) self._t = int(time.time()) self.env.set_tx_timestamp(self._t) self.env.set_flags(self.envsetflags, 1) self.env.open(self.homeDir, self.envflags | db.DB_CREATE) self.filename = "test" # Yes, a bare except is intended, since we're re-raising the exc. except: test_support.rmtree(self.homeDir) raise else: self.env = None self.filename = get_new_database_path() # create and open the DB self.d = db.DB(self.env) if not self.useEnv : if db.version() >= (4, 2) : self.d.set_cachesize(*self.cachesize) cachesize = self.d.get_cachesize() self.assertEqual(cachesize[0], self.cachesize[0]) self.assertEqual(cachesize[2], self.cachesize[2]) # Berkeley DB expands the cache 25% accounting overhead, # if the cache is small. self.assertEqual(125, int(100.0*cachesize[1]/self.cachesize[1])) self.d.set_flags(self.dbsetflags) if self.dbname: self.d.open(self.filename, self.dbname, self.dbtype, self.dbopenflags|db.DB_CREATE, self.dbmode) else: self.d.open(self.filename, # try out keyword args mode = self.dbmode, dbtype = self.dbtype, flags = self.dbopenflags|db.DB_CREATE) if not self.useEnv: self.assertRaises(db.DBInvalidArgError, self.d.set_cachesize, *self.cachesize) self.populateDB()
def test04_lock_timeout2(self): self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT) self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT) self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT) self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT) def deadlock_detection() : while not deadlock_detection.end : deadlock_detection.count = \ self.env.lock_detect(db.DB_LOCK_EXPIRE) if deadlock_detection.count : while not deadlock_detection.end : pass break time.sleep(0.01) deadlock_detection.end=False deadlock_detection.count=0 t=Thread(target=deadlock_detection) import sys if sys.version_info[0] < 3 : t.setDaemon(True) else : t.daemon = True t.start() self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT) anID = self.env.lock_id() anID2 = self.env.lock_id() self.assertNotEqual(anID, anID2) lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE) start_time=time.time() self.assertRaises(db.DBLockNotGrantedError, self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ) end_time=time.time() deadlock_detection.end=True # Floating point rounding self.assertTrue((end_time-start_time) >= 0.0999) self.env.lock_put(lock) t.join() self.env.lock_id_free(anID) self.env.lock_id_free(anID2) if db.version() >= (4,6): self.assertTrue(deadlock_detection.count>0)
def test_suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(AssociateErrorTestCase)) suite.addTest(unittest.makeSuite(AssociateHashTestCase)) suite.addTest(unittest.makeSuite(AssociateBTreeTestCase)) suite.addTest(unittest.makeSuite(AssociateRecnoTestCase)) if db.version() >= (4, 1): suite.addTest(unittest.makeSuite(AssociateBTreeTxnTestCase)) suite.addTest(unittest.makeSuite(ShelveAssociateHashTestCase)) suite.addTest(unittest.makeSuite(ShelveAssociateBTreeTestCase)) suite.addTest(unittest.makeSuite(ShelveAssociateRecnoTestCase)) if have_threads: suite.addTest(unittest.makeSuite(ThreadedAssociateHashTestCase)) suite.addTest(unittest.makeSuite(ThreadedAssociateBTreeTestCase)) suite.addTest(unittest.makeSuite(ThreadedAssociateRecnoTestCase)) return suite
def test06_close_txn_before_dup_cursor(self) : dbenv = db.DBEnv() dbenv.open(self.homeDir,db.DB_INIT_TXN | db.DB_INIT_MPOOL | db.DB_INIT_LOG | db.DB_CREATE) d = db.DB(dbenv) txn = dbenv.txn_begin() if db.version() < (4,1) : d.open(self.filename, dbtype = db.DB_HASH, flags = db.DB_CREATE) else : d.open(self.filename, dbtype = db.DB_HASH, flags = db.DB_CREATE, txn=txn) d.put("XXX", "yyy", txn=txn) txn.commit() txn = dbenv.txn_begin() c1 = d.cursor(txn) c2 = c1.dup() self.assertEquals(("XXX", "yyy"), c1.first()) import warnings # Not interested in warnings about implicit close. with warnings.catch_warnings(): warnings.simplefilter("ignore") txn.commit() self.assertRaises(db.DBCursorClosedError, c2.first)
def _create_env(self, must_open_db) : self.dbenv = db.DBEnv() self.dbenv.set_tx_max(self.num_txns) self.dbenv.set_lk_max_lockers(self.num_txns*2) self.dbenv.set_lk_max_locks(self.num_txns*2) self.dbenv.set_lk_max_objects(self.num_txns*2) if self.nosync : self.dbenv.set_flags(db.DB_TXN_NOSYNC,True) self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_THREAD | db.DB_RECOVER | db.DB_INIT_TXN | db.DB_INIT_LOG | db.DB_INIT_MPOOL | db.DB_INIT_LOCK, 0666) self.db = db.DB(self.dbenv) self.db.set_re_len(db.DB_GID_SIZE) if must_open_db : if db.version() >= (4,2) : txn=self.dbenv.txn_begin() self.db.open(self.filename, db.DB_QUEUE, db.DB_CREATE | db.DB_THREAD, 0666, txn=txn) txn.commit() else : self.db.open(self.filename, db.DB_QUEUE, db.DB_CREATE | db.DB_THREAD, 0666)
def test02_basicPost32(self): # Basic Queue tests using the new DB.consume method in DB 3.2+ # (No cursor needed) if verbose: print '\n', '-=' * 30 print "Running %s.test02_basicPost32..." % self.__class__.__name__ if db.version() < (3, 2, 0): if verbose: print "Test not run, DB not new enough..." return d = db.DB() d.set_re_len(40) # Queues must be fixed length d.open(self.filename, db.DB_QUEUE, db.DB_CREATE) if verbose: print "before appends" + '-' * 30 pprint(d.stat()) for x in string.letters: d.append(x * 40) self.assertEqual(len(d), len(string.letters)) d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") self.assertEqual(len(d), len(string.letters) + 3) if verbose: print "before close" + '-' * 30 pprint(d.stat()) d.close() del d d = db.DB() d.open(self.filename) #d.set_get_returns_none(true) if verbose: print "after open" + '-' * 30 pprint(d.stat()) d.append("one more") if verbose: print "after append" + '-' * 30 pprint(d.stat()) rec = d.consume() while rec: if verbose: print rec rec = d.consume() if verbose: print "after consume loop" + '-' * 30 pprint(d.stat()) d.close()
def test02_basicPost32(self): # Basic Queue tests using the new DB.consume method in DB 3.2+ # (No cursor needed) if verbose: print '\n', '-=' * 30 print "Running %s.test02_basicPost32..." % self.__class__.__name__ if db.version() < (3, 2, 0): if verbose: print "Test not run, DB not new enough..." return d = db.DB() d.set_re_len(40) # Queues must be fixed length d.open(self.filename, db.DB_QUEUE, db.DB_CREATE) if verbose: print "before appends" + '-' * 30 pprint(d.stat()) for x in string.letters: d.append(x * 40) self.assertEqual(len(d), len(string.letters)) d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") self.assertEqual(len(d), len(string.letters)+3) if verbose: print "before close" + '-' * 30 pprint(d.stat()) d.close() del d d = db.DB() d.open(self.filename) #d.set_get_returns_none(true) if verbose: print "after open" + '-' * 30 pprint(d.stat()) d.append("one more") if verbose: print "after append" + '-' * 30 pprint(d.stat()) rec = d.consume() while rec: if verbose: print rec rec = d.consume() if verbose: print "after consume loop" + '-' * 30 pprint(d.stat()) d.close()
class DBBaseReplication(DBReplication): def setUp(self): DBReplication.setUp(self) def confirmed_master(a, b, c): if b == db.DB_EVENT_REP_MASTER or b == db.DB_EVENT_REP_ELECTED: self.confirmed_master = True def client_startupdone(a, b, c): if b == db.DB_EVENT_REP_STARTUPDONE: self.client_startupdone = True self.dbenvMaster.set_event_notify(confirmed_master) self.dbenvClient.set_event_notify(client_startupdone) import Queue self.m2c = Queue.Queue() self.c2m = Queue.Queue() def m2c(dbenv, control, rec, lsnp, envid, flags): self.m2c.put((control, rec)) def c2m(dbenv, control, rec, lsnp, envid, flags): self.c2m.put((control, rec)) self.dbenvMaster.rep_set_transport(13, m2c) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_transport(3, c2m) self.dbenvClient.rep_set_priority(0) self.assertEqual(self.dbenvMaster.rep_get_priority(), 10) self.assertEqual(self.dbenvClient.rep_get_priority(), 0) def thread_master(): return self.thread_do(self.dbenvMaster, self.c2m, 3, self.master_doing_election, True) def thread_client(): return self.thread_do(self.dbenvClient, self.m2c, 13, self.client_doing_election, False) from threading import Thread t_m = Thread(target=thread_master) t_c = Thread(target=thread_client) import sys if sys.version_info[0] < 3: t_m.setDaemon(True) t_c.setDaemon(True) else: t_m.daemon = True t_c.daemon = True self.t_m = t_m self.t_c = t_c self.dbMaster = self.dbClient = None self.master_doing_election = [False] self.client_doing_election = [False] return def tearDown(self): if self.dbClient: self.dbClient.close() if self.dbMaster: self.dbMaster.close() self.m2c.put(None) self.c2m.put(None) self.t_m.join() self.t_c.join() def dummy(*args): pass self.dbenvMaster.set_event_notify(dummy) self.dbenvClient.set_event_notify(dummy) self.dbenvMaster.rep_set_transport(13, dummy) self.dbenvClient.rep_set_transport(3, dummy) self.dbenvClient.close() self.dbenvMaster.close() test_support.rmtree(self.homeDirClient) test_support.rmtree(self.homeDirMaster) return def basic_rep_threading(self): self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER) self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT) def thread_do(env, q, envid, election_status, must_be_master): while True: v = q.get() if v is None: return env.rep_process_message(v[0], v[1], envid) return self.thread_do = thread_do self.t_m.start() self.t_c.start() def test01_basic_replication(self): self.basic_rep_threading() import time timeout = time.time() + 60 while time.time() < timeout and not (self.confirmed_master and self.client_startupdone): time.sleep(0.02) self.assertTrue(time.time() < timeout) self.dbMaster = db.DB(self.dbenvMaster) txn = self.dbenvMaster.txn_begin() self.dbMaster.open('test', db.DB_HASH, db.DB_CREATE, 438, txn=txn) txn.commit() import time, os.path timeout = time.time() + 10 while time.time() < timeout and not os.path.exists( os.path.join(self.homeDirClient, 'test')): time.sleep(0.01) self.dbClient = db.DB(self.dbenvClient) while True: txn = self.dbenvClient.txn_begin() try: self.dbClient.open('test', db.DB_HASH, flags=db.DB_RDONLY, mode=438, txn=txn) except db.DBRepHandleDeadError: txn.abort() self.dbClient.close() self.dbClient = db.DB(self.dbenvClient) continue txn.commit() break d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR) self.assertTrue('master_changes' in d) txn = self.dbenvMaster.txn_begin() self.dbMaster.put('ABC', '123', txn=txn) txn.commit() import time timeout = time.time() + 10 v = None while time.time() < timeout and v is None: txn = self.dbenvClient.txn_begin() v = self.dbClient.get('ABC', txn=txn) txn.commit() if v is None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEqual('123', v) txn = self.dbenvMaster.txn_begin() self.dbMaster.delete('ABC', txn=txn) txn.commit() timeout = time.time() + 10 while time.time() < timeout and v is not None: txn = self.dbenvClient.txn_begin() v = self.dbClient.get('ABC', txn=txn) txn.commit() if v is None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEqual(None, v) return if db.version() >= (4, 7): def test02_test_request(self): self.basic_rep_threading() minimum, maximum = self.dbenvClient.rep_get_request() self.dbenvClient.rep_set_request(minimum - 1, maximum + 1) self.assertEqual(self.dbenvClient.rep_get_request(), (minimum - 1, maximum + 1)) if db.version() >= (4, 6): def test03_master_election(self): self.dbenvMaster.rep_start(flags=db.DB_REP_CLIENT) self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT) def thread_do(env, q, envid, election_status, must_be_master): while True: v = q.get() if v is None: return r = env.rep_process_message(v[0], v[1], envid) if must_be_master and self.confirmed_master: self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER) must_be_master = False if r[0] == db.DB_REP_HOLDELECTION: def elect(): while True: try: env.rep_elect(2, 1) election_status[0] = False break except db.DBRepUnavailError: pass if not election_status[0] and not self.confirmed_master: from threading import Thread election_status[0] = True t = Thread(target=elect) import sys if sys.version_info[0] < 3: t.setDaemon(True) else: t.daemon = True t.start() return self.thread_do = thread_do self.t_m.start() self.t_c.start() self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000) self.client_doing_election[0] = True while True: try: self.dbenvClient.rep_elect(2, 1) self.client_doing_election[0] = False break except db.DBRepUnavailError: pass self.assertTrue(self.confirmed_master) while not self.dbenvClient.rep_stat()['startup_complete']: pass if db.version() >= (4, 7): def test04_test_clockskew(self): fast, slow = (1234, 1230) self.dbenvMaster.rep_set_clockskew(fast, slow) self.assertEqual((fast, slow), self.dbenvMaster.rep_get_clockskew()) self.basic_rep_threading()
def test01_basic_replication(self): master_port = test_support.find_unused_port() client_port = test_support.find_unused_port() if db.version() >= (5, 2): self.site = self.dbenvMaster.repmgr_site("127.0.0.1", master_port) self.site.set_config(db.DB_GROUP_CREATOR, True) self.site.set_config(db.DB_LOCAL_SITE, True) self.site2 = self.dbenvMaster.repmgr_site("127.0.0.1", client_port) self.site3 = self.dbenvClient.repmgr_site("127.0.0.1", master_port) self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True) self.site4 = self.dbenvClient.repmgr_site("127.0.0.1", client_port) self.site4.set_config(db.DB_LOCAL_SITE, True) d = { db.DB_BOOTSTRAP_HELPER: [False, False, True, False], db.DB_GROUP_CREATOR: [True, False, False, False], db.DB_LEGACY: [False, False, False, False], db.DB_LOCAL_SITE: [True, False, False, True], db.DB_REPMGR_PEER: [False, False, False, False], } for i, j in d.items(): for k, v in zip([self.site, self.site2, self.site3, self.site4], j): if v: self.assertTrue(k.get_config(i)) else: self.assertFalse(k.get_config(i)) self.assertNotEqual(self.site.get_eid(), self.site2.get_eid()) self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid()) for i, j in zip( [self.site, self.site2, self.site3, self.site4], [master_port, client_port, master_port, client_port] ): addr = i.get_address() self.assertEqual(addr, ("127.0.0.1", j)) for i in [self.site, self.site2]: self.assertEqual(i.get_address(), self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address()) for i in [self.site3, self.site4]: self.assertEqual(i.get_address(), self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address()) else: self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321) self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123) self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345) self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER) self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT) self.assertEqual(self.dbenvMaster.rep_get_nsites(), 2) self.assertEqual(self.dbenvClient.rep_get_nsites(), 2) self.assertEqual(self.dbenvMaster.rep_get_priority(), 10) self.assertEqual(self.dbenvClient.rep_get_priority(), 0) self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time() + 10 while (time.time() < timeout) and not (self.confirmed_master and self.client_startupdone): time.sleep(0.02) self.assertTrue(time.time() < timeout) d = self.dbenvMaster.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] # There is only one self.assertEqual(d[0], "127.0.0.1") self.assertEqual(d[1], client_port) self.assertTrue((d[2] == db.DB_REPMGR_CONNECTED) or (d[2] == db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] # There is only one self.assertEqual(d[0], "127.0.0.1") self.assertEqual(d[1], master_port) self.assertTrue((d[2] == db.DB_REPMGR_CONNECTED) or (d[2] == db.DB_REPMGR_DISCONNECTED)) if db.version() >= (4, 6): d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR) self.assertTrue("msgs_queued" in d) self.dbMaster = db.DB(self.dbenvMaster) txn = self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time, os.path timeout = time.time() + 10 while (time.time() < timeout) and not (os.path.exists(os.path.join(self.homeDirClient, "test"))): time.sleep(0.01) self.dbClient = db.DB(self.dbenvClient) while True: txn = self.dbenvClient.txn_begin() try: self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError: txn.abort() self.dbClient.close() self.dbClient = db.DB(self.dbenvClient) continue txn.commit() break txn = self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout = time.time() + 10 v = None while (time.time() < timeout) and (v is None): txn = self.dbenvClient.txn_begin() v = self.dbClient.get("ABC", txn=txn) txn.commit() if v is None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEqual("123", v) txn = self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout = time.time() + 10 while (time.time() < timeout) and (v is not None): txn = self.dbenvClient.txn_begin() v = self.dbClient.get("ABC", txn=txn) txn.commit() if v is None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEqual(None, v)
def test01_basic_replication(self): master_port = test_support.find_unused_port() self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) client_port = test_support.find_unused_port() self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321) self.assertEquals( self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123) self.assertEquals( self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEquals( self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEquals( self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEquals( self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345) self.assertEquals( self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER) self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT) self.assertEquals(self.dbenvMaster.rep_get_nsites(), 2) self.assertEquals(self.dbenvClient.rep_get_nsites(), 2) self.assertEquals(self.dbenvMaster.rep_get_priority(), 10) self.assertEquals(self.dbenvClient.rep_get_priority(), 0) self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time() + 60 while (time.time() < timeout) and not (self.confirmed_master and self.client_startupdone): time.sleep(0.02) # this fails on Windows as self.client_startupdone never gets set # to True - see bug 3892. BUT - even though this assertion # fails on Windows the rest of the test passes - so to prove # that we let the rest of the test run. Sadly we can't # make use of raising TestSkipped() here (unittest still # reports it as an error), so we yell to stderr. import sys if sys.platform == "win32": print >> sys.stderr, \ "XXX - windows bsddb replication fails on windows and is skipped" print >> sys.stderr, "XXX - Please see issue #3892" # It also fails irregularly on other platforms, and again the # rest of the tests pass. Since bsddb support is unmaintained, and # is gone in py3k, we just emit a warning instead of a test failure # so as to improve buildbot stability. elif time.time() > timeout: print >> sys.stderr, \ "XXX - timeout before startup confirmed, see issue #3892." d = self.dbenvMaster.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], client_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], master_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) if db.version() >= (4, 6): d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR) self.assertTrue("msgs_queued" in d) self.dbMaster = db.DB(self.dbenvMaster) txn = self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time, os.path timeout = time.time() + 10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient = db.DB(self.dbenvClient) while True: txn = self.dbenvClient.txn_begin() try: self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError: txn.abort() self.dbClient.close() self.dbClient = db.DB(self.dbenvClient) continue txn.commit() break txn = self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout = time.time() + 10 v = None while (time.time() < timeout) and (v == None): txn = self.dbenvClient.txn_begin() v = self.dbClient.get("ABC", txn=txn) txn.commit() if v == None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEquals("123", v) txn = self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout = time.time() + 10 while (time.time() < timeout) and (v != None): txn = self.dbenvClient.txn_begin() v = self.dbClient.get("ABC", txn=txn) txn.commit() if v == None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEquals(None, v)
self.assertTrue(size <= size2[1]) self.assertTrue(2*size > size2[1]) if db.version() >= (4, 4) : def test_mutex_stat(self) : self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK) stat = self.env.mutex_stat() self.assertTrue("mutex_inuse_max" in stat) def test_lg_filemode(self) : for i in [0600, 0660, 0666] : self.env.set_lg_filemode(i) self.assertEqual(i, self.env.get_lg_filemode()) if db.version() >= (4, 3) : def test_mp_max_openfd(self) : for i in [17, 31, 42] : self.env.set_mp_max_openfd(i) self.assertEqual(i, self.env.get_mp_max_openfd()) def test_mp_max_write(self) : for i in [100, 200, 300] : for j in [1, 2, 3] : j *= 1000000 self.env.set_mp_max_write(i, j) v=self.env.get_mp_max_write() self.assertEqual((i, j), v) if db.version() >= (4, 2) : def test_invalid_txn(self) :
def test01_basic_replication(self) : self.basic_rep_threading() # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time()+10 while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.dbMaster=db.DB(self.dbenvMaster) txn=self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time,os.path timeout=time.time()+10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient=db.DB(self.dbenvClient) while True : txn=self.dbenvClient.txn_begin() try : self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError : txn.abort() self.dbClient.close() self.dbClient=db.DB(self.dbenvClient) continue txn.commit() break if db.version() >= (4,1) : d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR); self.assertTrue("master_changes" in d) txn=self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout=time.time()+10 v=None while (time.time()<timeout) and (v==None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals("123", v) txn=self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout=time.time()+10 while (time.time()<timeout) and (v!=None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals(None, v)
self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=1) def test03c_SimpleCursorGetReturnsNone2(self): # same test but raise exceptions instead of returning None if verbose: print '\n', '-=' * 30 print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \ self.__class__.__name__ old = self.d.set_get_returns_none(1) self.assertEqual(old, 2) old = self.d.set_get_returns_none(2) self.assertEqual(old, 1) self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0) if db.version() >= (4, 6): def test03d_SimpleCursorPriority(self): c = self.d.cursor() c.set_priority(db.DB_PRIORITY_VERY_LOW) # Positional self.assertEqual(db.DB_PRIORITY_VERY_LOW, c.get_priority()) c.set_priority(priority=db.DB_PRIORITY_HIGH) # Keyword self.assertEqual(db.DB_PRIORITY_HIGH, c.get_priority()) c.close() #---------------------------------------- def test04_PartialGetAndPut(self): d = self.d if verbose: print '\n', '-=' * 30
shm_key=137 self.env.set_shm_key(shm_key) self.assertEqual(shm_key, self.env.get_shm_key()) self.env.set_shm_key(shm_key+1) self.assertEqual(shm_key+1, self.env.get_shm_key()) # You can not change configuration after opening # the environment. self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) # If we try to reconfigure cache after opening the # environment, core dump. self.assertRaises(db.DBInvalidArgError, self.env.set_shm_key, shm_key) self.assertEqual(shm_key+1, self.env.get_shm_key()) if db.version() >= (4, 4) : def test_mutex_setget_max(self) : v = self.env.mutex_get_max() v2 = v*2+1 self.env.mutex_set_max(v2) self.assertEqual(v2, self.env.mutex_get_max()) self.env.mutex_set_max(v) self.assertEqual(v, self.env.mutex_get_max()) # You can not change configuration after opening # the environment. self.env.open(self.homeDir, db.DB_CREATE) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_max, v2)
def _destroy_env(self): if self.nosync or (db.version()[:2] == (4, 6)): # Known bug self.dbenv.log_flush() self.db.close() self.dbenv.close()
class DBEnvClosedEarlyCrash(unittest.TestCase): def setUp(self): self.homeDir = get_new_environment_path() self.filename = "test" def tearDown(self): test_support.rmtree(self.homeDir) def test01_close_dbenv_before_db(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) d = db.DB(dbenv) d2 = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) self.assertRaises(db.DBNoSuchFileError, d2.open, self.filename + "2", db.DB_BTREE, db.DB_THREAD, 0666) d.put("test", "this is a test") self.assertEqual(d.get("test"), "this is a test", "put!=get") dbenv.close() # This "close" should close the child db handle also self.assertRaises(db.DBError, d.get, "test") def test02_close_dbenv_before_dbcursor(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) d = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) d.put("test", "this is a test") d.put("test2", "another test") d.put("test3", "another one") self.assertEqual(d.get("test"), "this is a test", "put!=get") c = d.cursor() c.first() c.next() d.close() # This "close" should close the child db handle also # db.close should close the child cursor self.assertRaises(db.DBError, c.next) d = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) c = d.cursor() c.first() c.next() dbenv.close() # The "close" should close the child db handle also, with cursors self.assertRaises(db.DBError, c.next) def test03_close_db_before_dbcursor_without_env(self): import os.path path = os.path.join(self.homeDir, self.filename) d = db.DB() d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) d.put("test", "this is a test") d.put("test2", "another test") d.put("test3", "another one") self.assertEqual(d.get("test"), "this is a test", "put!=get") c = d.cursor() c.first() c.next() d.close() # The "close" should close the child db handle also self.assertRaises(db.DBError, c.next) def test04_close_massive(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) dbs = [db.DB(dbenv) for i in xrange(16)] cursors = [] for i in dbs: i.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) dbs[10].put("test", "this is a test") dbs[10].put("test2", "another test") dbs[10].put("test3", "another one") self.assertEqual(dbs[4].get("test"), "this is a test", "put!=get") for i in dbs: cursors.extend([i.cursor() for j in xrange(32)]) for i in dbs[::3]: i.close() for i in cursors[::3]: i.close() # Check for missing exception in DB! (after DB close) self.assertRaises(db.DBError, dbs[9].get, "test") # Check for missing exception in DBCursor! (after DB close) self.assertRaises(db.DBError, cursors[101].first) cursors[80].first() cursors[80].next() dbenv.close() # This "close" should close the child db handle also # Check for missing exception! (after DBEnv close) self.assertRaises(db.DBError, cursors[80].next) def test05_close_dbenv_delete_db_success(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) d = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) dbenv.close() # This "close" should close the child db handle also del d try: import gc except ImportError: gc = None if gc: # force d.__del__ [DB_dealloc] to be called gc.collect() def test06_close_txn_before_dup_cursor(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_TXN | db.DB_INIT_MPOOL | db.DB_INIT_LOG | db.DB_CREATE) d = db.DB(dbenv) txn = dbenv.txn_begin() d.open(self.filename, dbtype=db.DB_HASH, flags=db.DB_CREATE, txn=txn) d.put("XXX", "yyy", txn=txn) txn.commit() txn = dbenv.txn_begin() c1 = d.cursor(txn) c2 = c1.dup() self.assertEquals(("XXX", "yyy"), c1.first()) # Not interested in warnings about implicit close. import warnings if sys.version_info < (2, 6): # Completely resetting the warning state is # problematic with python >=2.6 with -3 (py3k warning), # because some stdlib modules selectively ignores warnings. warnings.simplefilter("ignore") txn.commit() warnings.resetwarnings() else: # When we drop support for python 2.3 and 2.4 # we could use: (in 2.5 we need a __future__ statement) # # with warnings.catch_warnings(): # warnings.simplefilter("ignore") # txn.commit() # # We can not use "with" as is, because it would be invalid syntax # in python 2.3, 2.4 and (with no __future__) 2.5. # Here we simulate "with" following PEP 343 : w = warnings.catch_warnings() w.__enter__() try: warnings.simplefilter("ignore") txn.commit() finally: w.__exit__() self.assertRaises(db.DBCursorClosedError, c2.first) if db.version() > (4, 3, 0): def test07_close_db_before_sequence(self): import os.path path = os.path.join(self.homeDir, self.filename) d = db.DB() d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) dbs = db.DBSequence(d) d.close() # This "close" should close the child DBSequence also dbs.close() # If not closed, core dump (in Berkeley DB 4.6.*)
shm_key = 137 self.env.set_shm_key(shm_key) self.assertEqual(shm_key, self.env.get_shm_key()) self.env.set_shm_key(shm_key + 1) self.assertEqual(shm_key + 1, self.env.get_shm_key()) # You can not change configuration after opening # the environment. self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) # If we try to reconfigure cache after opening the # environment, core dump. self.assertRaises(db.DBInvalidArgError, self.env.set_shm_key, shm_key) self.assertEqual(shm_key + 1, self.env.get_shm_key()) if db.version() >= (4, 4): def test_mutex_setget_max(self): v = self.env.mutex_get_max() v2 = v * 2 + 1 self.env.mutex_set_max(v2) self.assertEqual(v2, self.env.mutex_get_max()) self.env.mutex_set_max(v) self.assertEqual(v, self.env.mutex_get_max()) # You can not change configuration after opening # the environment. self.env.open(self.homeDir, db.DB_CREATE) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_max, v2)
class AssociateTestCase(unittest.TestCase): keytype = '' envFlags = 0 dbFlags = 0 def setUp(self): self.filename = self.__class__.__name__ + '.db' self.homeDir = get_new_environment_path() self.env = db.DBEnv() self.env.open( self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_THREAD | self.envFlags) def tearDown(self): self.closeDB() self.env.close() self.env = None test_support.rmtree(self.homeDir) def addDataToDB(self, d, txn=None): for key, value in musicdata.items(): if type(self.keytype) == type(''): key = "%02d" % key d.put(key, '|'.join(value), txn=txn) def createDB(self, txn=None): self.cur = None self.secDB = None self.primary = db.DB(self.env) self.primary.set_get_returns_none(2) self.primary.open(self.filename, "primary", self.dbtype, db.DB_CREATE | db.DB_THREAD | self.dbFlags, txn=txn) def closeDB(self): if self.cur: self.cur.close() self.cur = None if self.secDB: self.secDB.close() self.secDB = None self.primary.close() self.primary = None def getDB(self): return self.primary def _associateWithDB(self, getGenre): self.createDB() self.secDB = db.DB(self.env) self.secDB.set_flags(db.DB_DUP) self.secDB.set_get_returns_none(2) self.secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE | db.DB_THREAD | self.dbFlags) self.getDB().associate(self.secDB, getGenre) self.addDataToDB(self.getDB()) self.finish_test(self.secDB) def test01_associateWithDB(self): if verbose: print '\n', '-=' * 30 print "Running %s.test01_associateWithDB..." % \ self.__class__.__name__ return self._associateWithDB(self.getGenre) def _associateAfterDB(self, getGenre): self.createDB() self.addDataToDB(self.getDB()) self.secDB = db.DB(self.env) self.secDB.set_flags(db.DB_DUP) self.secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE | db.DB_THREAD | self.dbFlags) # adding the DB_CREATE flag will cause it to index existing records self.getDB().associate(self.secDB, getGenre, db.DB_CREATE) self.finish_test(self.secDB) def test02_associateAfterDB(self): if verbose: print '\n', '-=' * 30 print "Running %s.test02_associateAfterDB..." % \ self.__class__.__name__ return self._associateAfterDB(self.getGenre) if db.version() >= (4, 6): def test03_associateWithDB(self): if verbose: print '\n', '-=' * 30 print "Running %s.test03_associateWithDB..." % \ self.__class__.__name__ return self._associateWithDB(self.getGenreList) def test04_associateAfterDB(self): if verbose: print '\n', '-=' * 30 print "Running %s.test04_associateAfterDB..." % \ self.__class__.__name__ return self._associateAfterDB(self.getGenreList) def finish_test(self, secDB, txn=None): # 'Blues' should not be in the secondary database vals = secDB.pget('Blues', txn=txn) self.assertEqual(vals, None, vals) vals = secDB.pget('Unknown', txn=txn) self.assert_(vals[0] == 99 or vals[0] == '99', vals) vals[1].index('Unknown') vals[1].index('Unnamed') vals[1].index('unknown') if verbose: print "Primary key traversal:" self.cur = self.getDB().cursor(txn) count = 0 rec = self.cur.first() while rec is not None: if type(self.keytype) == type(''): self.assert_(int(rec[0])) # for primary db, key is a number else: self.assert_(rec[0] and type(rec[0]) == type(0)) count = count + 1 if verbose: print rec rec = getattr(self.cur, "next")() self.assertEqual(count, len(musicdata)) # all items accounted for if verbose: print "Secondary key traversal:" self.cur = secDB.cursor(txn) count = 0 # test cursor pget vals = self.cur.pget('Unknown', flags=db.DB_LAST) self.assert_(vals[1] == 99 or vals[1] == '99', vals) self.assertEqual(vals[0], 'Unknown') vals[2].index('Unknown') vals[2].index('Unnamed') vals[2].index('unknown') vals = self.cur.pget('Unknown', data='wrong value', flags=db.DB_GET_BOTH) self.assertEqual(vals, None, vals) rec = self.cur.first() self.assertEqual(rec[0], "Jazz") while rec is not None: count = count + 1 if verbose: print rec rec = getattr(self.cur, "next")() # all items accounted for EXCEPT for 1 with "Blues" genre self.assertEqual(count, len(musicdata) - 1) self.cur = None def getGenre(self, priKey, priData): self.assertEqual(type(priData), type("")) genre = priData.split('|')[2] if verbose: print 'getGenre key: %r data: %r' % (priKey, priData) if genre == 'Blues': return db.DB_DONOTINDEX else: return genre def getGenreList(self, priKey, PriData): v = self.getGenre(priKey, PriData) if type(v) == type(""): v = [v] return v
def test_suite(): suite = unittest.TestSuite() if db.version() >= (4, 4): suite.addTest(unittest.makeSuite(FileidResetTestCase)) return suite
def test01_basic_replication(self) : master_port = test_support.find_unused_port() client_port = test_support.find_unused_port() if db.version() >= (5, 2) : self.site = self.dbenvMaster.repmgr_site("127.0.0.1", master_port) self.site.set_config(db.DB_GROUP_CREATOR, True) self.site.set_config(db.DB_LOCAL_SITE, True) self.site2 = self.dbenvMaster.repmgr_site("127.0.0.1", client_port) self.site3 = self.dbenvClient.repmgr_site("127.0.0.1", master_port) self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True) self.site4 = self.dbenvClient.repmgr_site("127.0.0.1", client_port) self.site4.set_config(db.DB_LOCAL_SITE, True) d = { db.DB_BOOTSTRAP_HELPER: [False, False, True, False], db.DB_GROUP_CREATOR: [True, False, False, False], db.DB_LEGACY: [False, False, False, False], db.DB_LOCAL_SITE: [True, False, False, True], db.DB_REPMGR_PEER: [False, False, False, False ], } for i, j in d.items() : for k, v in \ zip([self.site, self.site2, self.site3, self.site4], j) : if v : self.assertTrue(k.get_config(i)) else : self.assertFalse(k.get_config(i)) self.assertNotEqual(self.site.get_eid(), self.site2.get_eid()) self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid()) for i, j in zip([self.site, self.site2, self.site3, self.site4], \ [master_port, client_port, master_port, client_port]) : addr = i.get_address() self.assertEqual(addr, ("127.0.0.1", j)) for i in [self.site, self.site2] : self.assertEqual(i.get_address(), self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address()) for i in [self.site3, self.site4] : self.assertEqual(i.get_address(), self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address()) else : self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321) self.assertEqual(self.dbenvMaster.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100123) self.assertEqual(self.dbenvClient.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEqual(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEqual(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEqual(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100345) self.assertEqual(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER); self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT); self.assertEqual(self.dbenvMaster.rep_get_nsites(),2) self.assertEqual(self.dbenvClient.rep_get_nsites(),2) self.assertEqual(self.dbenvMaster.rep_get_priority(),10) self.assertEqual(self.dbenvClient.rep_get_priority(),0) self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time()+60 while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : time.sleep(0.02) # self.client_startupdone does not always get set to True within # the timeout. On windows this may be a deep issue, on other # platforms it is likely just a timing issue, especially on slow # virthost buildbots (see issue 3892 for more). Even though # the timeout triggers, the rest of this test method usually passes # (but not all of it always, see below). So we just note the # timeout on stderr and keep soldering on. if time.time()>timeout: import sys print >> sys.stderr, ("XXX: timeout happened before" "startup was confirmed - see issue 3892") startup_timeout = True d = self.dbenvMaster.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] # There is only one self.assertEqual(d[0], "127.0.0.1") self.assertEqual(d[1], client_port) self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \ (d[2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] # There is only one self.assertEqual(d[0], "127.0.0.1") self.assertEqual(d[1], master_port) self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \ (d[2]==db.DB_REPMGR_DISCONNECTED)) if db.version() >= (4,6) : d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR); self.assertTrue("msgs_queued" in d) self.dbMaster=db.DB(self.dbenvMaster) txn=self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time,os.path timeout=time.time()+10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient=db.DB(self.dbenvClient) while True : txn=self.dbenvClient.txn_begin() try : self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError : txn.abort() self.dbClient.close() self.dbClient=db.DB(self.dbenvClient) continue txn.commit() break txn=self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout=time.time()+10 v=None while (time.time()<timeout) and (v is None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v is None : time.sleep(0.02) # If startup did not happen before the timeout above, then this test # sometimes fails. This happens randomly, which causes buildbot # instability, but all the other bsddb tests pass. Since bsddb3 in the # stdlib is currently not getting active maintenance, and is gone in # py3k, we just skip the end of the test in that case. if time.time()>=timeout and startup_timeout: self.skipTest("replication test skipped due to random failure, " "see issue 3892") self.assertTrue(time.time()<timeout) self.assertEqual("123", v) txn=self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout=time.time()+10 while (time.time()<timeout) and (v is not None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v is None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEqual(None, v)
def test01_basic_replication(self): master_port = test_support.find_unused_port() client_port = test_support.find_unused_port() if db.version() >= (5, 2): self.site = self.dbenvMaster.repmgr_site('127.0.0.1', master_port) self.site.set_config(db.DB_GROUP_CREATOR, True) self.site.set_config(db.DB_LOCAL_SITE, True) self.site2 = self.dbenvMaster.repmgr_site('127.0.0.1', client_port) self.site3 = self.dbenvClient.repmgr_site('127.0.0.1', master_port) self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True) self.site4 = self.dbenvClient.repmgr_site('127.0.0.1', client_port) self.site4.set_config(db.DB_LOCAL_SITE, True) d = { db.DB_BOOTSTRAP_HELPER: [False, False, True, False], db.DB_GROUP_CREATOR: [True, False, False, False], db.DB_LEGACY: [False, False, False, False], db.DB_LOCAL_SITE: [True, False, False, True], db.DB_REPMGR_PEER: [False, False, False, False] } for i, j in d.items(): for k, v in zip( [self.site, self.site2, self.site3, self.site4], j): if v: self.assertTrue(k.get_config(i)) else: self.assertFalse(k.get_config(i)) self.assertNotEqual(self.site.get_eid(), self.site2.get_eid()) self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid()) for i, j in zip( [self.site, self.site2, self.site3, self.site4], [master_port, client_port, master_port, client_port]): addr = i.get_address() self.assertEqual(addr, ('127.0.0.1', j)) for i in [self.site, self.site2]: self.assertEqual( i.get_address(), self.dbenvMaster.repmgr_site_by_eid( i.get_eid()).get_address()) for i in [self.site3, self.site4]: self.assertEqual( i.get_address(), self.dbenvClient.repmgr_site_by_eid( i.get_eid()).get_address()) else: self.dbenvMaster.repmgr_set_local_site('127.0.0.1', master_port) self.dbenvClient.repmgr_set_local_site('127.0.0.1', client_port) self.dbenvMaster.repmgr_add_remote_site('127.0.0.1', client_port) self.dbenvClient.repmgr_add_remote_site('127.0.0.1', master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321) self.assertEqual( self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123) self.assertEqual( self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEqual( self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEqual( self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEqual( self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345) self.assertEqual( self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER) self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT) self.assertEqual(self.dbenvMaster.rep_get_nsites(), 2) self.assertEqual(self.dbenvClient.rep_get_nsites(), 2) self.assertEqual(self.dbenvMaster.rep_get_priority(), 10) self.assertEqual(self.dbenvClient.rep_get_priority(), 0) self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) import time timeout = time.time() + 60 while time.time() < timeout and not (self.confirmed_master and self.client_startupdone): time.sleep(0.02) if time.time() > timeout: import sys print >> sys.stderr, 'XXX: timeout happened beforestartup was confirmed - see issue 3892' startup_timeout = True d = self.dbenvMaster.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] self.assertEqual(d[0], '127.0.0.1') self.assertEqual(d[1], client_port) self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED or d[2] == db.DB_REPMGR_DISCONNECTED) d = self.dbenvClient.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] self.assertEqual(d[0], '127.0.0.1') self.assertEqual(d[1], master_port) self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED or d[2] == db.DB_REPMGR_DISCONNECTED) if db.version() >= (4, 6): d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR) self.assertTrue('msgs_queued' in d) self.dbMaster = db.DB(self.dbenvMaster) txn = self.dbenvMaster.txn_begin() self.dbMaster.open('test', db.DB_HASH, db.DB_CREATE, 438, txn=txn) txn.commit() import time, os.path timeout = time.time() + 10 while time.time() < timeout and not os.path.exists( os.path.join(self.homeDirClient, 'test')): time.sleep(0.01) self.dbClient = db.DB(self.dbenvClient) while True: txn = self.dbenvClient.txn_begin() try: self.dbClient.open('test', db.DB_HASH, flags=db.DB_RDONLY, mode=438, txn=txn) except db.DBRepHandleDeadError: txn.abort() self.dbClient.close() self.dbClient = db.DB(self.dbenvClient) continue txn.commit() break txn = self.dbenvMaster.txn_begin() self.dbMaster.put('ABC', '123', txn=txn) txn.commit() import time timeout = time.time() + 10 v = None while time.time() < timeout and v is None: txn = self.dbenvClient.txn_begin() v = self.dbClient.get('ABC', txn=txn) txn.commit() if v is None: time.sleep(0.02) if time.time() >= timeout and startup_timeout: self.skipTest( 'replication test skipped due to random failure, see issue 3892' ) self.assertTrue(time.time() < timeout) self.assertEqual('123', v) txn = self.dbenvMaster.txn_begin() self.dbMaster.delete('ABC', txn=txn) txn.commit() timeout = time.time() + 10 while time.time() < timeout and v is not None: txn = self.dbenvClient.txn_begin() v = self.dbClient.get('ABC', txn=txn) txn.commit() if v is None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEqual(None, v) return
class BasicTransactionTestCase(BasicTestCase): if sys.version_info < ( 2, 7) or sys.version_info >= (3, 0) and sys.version_info < (3, 2): def assertIn(self, a, b, msg=None): return self.assertTrue(a in b, msg=msg) dbopenflags = db.DB_THREAD | db.DB_AUTO_COMMIT useEnv = 1 envflags = db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_TXN envsetflags = db.DB_AUTO_COMMIT def tearDown(self): self.txn.commit() BasicTestCase.tearDown(self) def populateDB(self): txn = self.env.txn_begin() BasicTestCase.populateDB(self, _txn=txn) self.txn = self.env.txn_begin() def test06_Transactions(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test06_Transactions...' % self.__class__.__name__ self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) self.assertEqual(d.get('new rec', txn=self.txn), 'this is a new record') self.txn.abort() self.assertEqual(d.get('new rec'), None) self.txn = self.env.txn_begin() self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) self.assertEqual(d.get('new rec', txn=self.txn), 'this is a new record') self.txn.commit() self.assertEqual(d.get('new rec'), 'this is a new record') self.txn = self.env.txn_begin() c = d.cursor(self.txn) rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec rec = c.next() self.assertEqual(count, self._numKeys + 1) c.close() self.txn.commit() self.env.txn_checkpoint(0, 0, 0) statDict = self.env.log_stat(0) self.assertIn('magic', statDict) self.assertIn('version', statDict) self.assertIn('cur_file', statDict) self.assertIn('region_nowait', statDict) logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG) self.assertNotEqual(logs, None) for log in logs: if verbose: print 'log file: ' + log logs = self.env.log_archive(db.DB_ARCH_REMOVE) self.assertTrue(not logs) self.txn = self.env.txn_begin() return if db.version() >= (4, 6): def test08_exists(self): txn = self.env.txn_begin() self.d.put('abcde', 'ABCDE', txn=txn) txn.commit() txn = self.env.txn_begin() self.assertTrue( self.d.exists('abcde', txn=txn) == True, 'DB->exists() returns wrong value') self.assertTrue( self.d.exists('x', txn=txn) == False, 'DB->exists() returns wrong value') txn.abort() def test09_TxnTruncate(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test09_TxnTruncate...' % self.__class__.__name__ d.put('abcde', 'ABCDE') txn = self.env.txn_begin() num = d.truncate(txn) self.assertTrue(num >= 1, 'truncate returned <= 0 on non-empty database') num = d.truncate(txn) self.assertEqual( num, 0, 'truncate on empty DB returned nonzero (%r)' % (num, )) txn.commit() def test10_TxnLateUse(self): txn = self.env.txn_begin() txn.abort() try: txn.abort() except db.DBError as e: pass else: raise RuntimeError, 'DBTxn.abort() called after DB_TXN no longer valid w/o an exception' txn = self.env.txn_begin() txn.commit() try: txn.commit() except db.DBError as e: pass else: raise RuntimeError, 'DBTxn.commit() called after DB_TXN no longer valid w/o an exception' if db.version() >= (4, 4): def test_txn_name(self): txn = self.env.txn_begin() self.assertEqual(txn.get_name(), '') txn.set_name('XXYY') self.assertEqual(txn.get_name(), 'XXYY') txn.set_name('') self.assertEqual(txn.get_name(), '') txn.abort() def test_txn_set_timeout(self): txn = self.env.txn_begin() txn.set_timeout(1234567, db.DB_SET_LOCK_TIMEOUT) txn.set_timeout(2345678, flags=db.DB_SET_TXN_TIMEOUT) txn.abort() def test_get_tx_max(self): self.assertEqual(self.env.get_tx_max(), 30) def test_get_tx_timestamp(self): self.assertEqual(self.env.get_tx_timestamp(), self._t)
class DBEnv_general(DBEnv): def test_get_open_flags(self): flags = db.DB_CREATE | db.DB_INIT_MPOOL self.env.open(self.homeDir, flags) self.assertEqual(flags, self.env.get_open_flags()) def test_get_open_flags2(self): flags = db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_THREAD self.env.open(self.homeDir, flags) self.assertEqual(flags, self.env.get_open_flags()) if db.version() >= (4, 7): def test_lk_partitions(self): for i in [10, 20, 40]: self.env.set_lk_partitions(i) self.assertEqual(i, self.env.get_lk_partitions()) def test_getset_intermediate_dir_mode(self): self.assertEqual(None, self.env.get_intermediate_dir_mode()) for mode in ['rwx------', 'rw-rw-rw-', 'rw-r--r--']: self.env.set_intermediate_dir_mode(mode) self.assertEqual(mode, self.env.get_intermediate_dir_mode()) self.assertRaises(db.DBInvalidArgError, self.env.set_intermediate_dir_mode, 'abcde') return if db.version() >= (4, 6): def test_thread(self): for i in [16, 100, 1000]: self.env.set_thread_count(i) self.assertEqual(i, self.env.get_thread_count()) def test_cache_max(self): for size in [64, 128]: size = size * 1024 * 1024 self.env.set_cache_max(0, size) size2 = self.env.get_cache_max() self.assertEqual(0, size2[0]) self.assertTrue(size <= size2[1]) self.assertTrue(2 * size > size2[1]) if db.version() >= (4, 4): def test_mutex_stat(self): self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK) stat = self.env.mutex_stat() self.assertTrue('mutex_inuse_max' in stat) def test_lg_filemode(self): for i in [384, 432, 438]: self.env.set_lg_filemode(i) self.assertEqual(i, self.env.get_lg_filemode()) def test_mp_max_openfd(self): for i in [17, 31, 42]: self.env.set_mp_max_openfd(i) self.assertEqual(i, self.env.get_mp_max_openfd()) def test_mp_max_write(self): for i in [100, 200, 300]: for j in [1, 2, 3]: j *= 1000000 self.env.set_mp_max_write(i, j) v = self.env.get_mp_max_write() self.assertEqual((i, j), v) def test_invalid_txn(self): self.assertRaises(db.DBInvalidArgError, self.env.txn_begin) def test_mp_mmapsize(self): for i in [16, 32, 64]: i *= 1048576 self.env.set_mp_mmapsize(i) self.assertEqual(i, self.env.get_mp_mmapsize()) def test_tmp_dir(self): for i in ['a', 'bb', 'ccc']: self.env.set_tmp_dir(i) self.assertEqual(i, self.env.get_tmp_dir()) def test_flags(self): self.env.set_flags(db.DB_AUTO_COMMIT, 1) self.assertEqual(db.DB_AUTO_COMMIT, self.env.get_flags()) self.env.set_flags(db.DB_TXN_NOSYNC, 1) self.assertEqual(db.DB_AUTO_COMMIT | db.DB_TXN_NOSYNC, self.env.get_flags()) self.env.set_flags(db.DB_AUTO_COMMIT, 0) self.assertEqual(db.DB_TXN_NOSYNC, self.env.get_flags()) self.env.set_flags(db.DB_TXN_NOSYNC, 0) self.assertEqual(0, self.env.get_flags()) def test_lk_max_objects(self): for i in [1000, 2000, 3000]: self.env.set_lk_max_objects(i) self.assertEqual(i, self.env.get_lk_max_objects()) def test_lk_max_locks(self): for i in [1000, 2000, 3000]: self.env.set_lk_max_locks(i) self.assertEqual(i, self.env.get_lk_max_locks()) def test_lk_max_lockers(self): for i in [1000, 2000, 3000]: self.env.set_lk_max_lockers(i) self.assertEqual(i, self.env.get_lk_max_lockers()) def test_lg_regionmax(self): for i in [128, 256, 1000]: i = i * 1024 * 1024 self.env.set_lg_regionmax(i) j = self.env.get_lg_regionmax() self.assertTrue(i <= j) self.assertTrue(2 * i > j) def test_lk_detect(self): flags = [ db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS, db.DB_LOCK_MINLOCKS, db.DB_LOCK_MINWRITE, db.DB_LOCK_OLDEST, db.DB_LOCK_RANDOM, db.DB_LOCK_YOUNGEST ] flags.append(db.DB_LOCK_MAXWRITE) for i in flags: self.env.set_lk_detect(i) self.assertEqual(i, self.env.get_lk_detect()) def test_lg_dir(self): for i in ['a', 'bb', 'ccc', 'dddd']: self.env.set_lg_dir(i) self.assertEqual(i, self.env.get_lg_dir()) def test_lg_bsize(self): log_size = 71680 self.env.set_lg_bsize(log_size) self.assertTrue(self.env.get_lg_bsize() >= log_size) self.assertTrue(self.env.get_lg_bsize() < 4 * log_size) self.env.set_lg_bsize(4 * log_size) self.assertTrue(self.env.get_lg_bsize() >= 4 * log_size) def test_setget_data_dirs(self): dirs = ('a', 'b', 'c', 'd') for i in dirs: self.env.set_data_dir(i) self.assertEqual(dirs, self.env.get_data_dirs()) def test_setget_cachesize(self): cachesize = (0, 536870912, 3) self.env.set_cachesize(*cachesize) self.assertEqual(cachesize, self.env.get_cachesize()) cachesize = (0, 1048576, 5) self.env.set_cachesize(*cachesize) cachesize2 = self.env.get_cachesize() self.assertEqual(cachesize[0], cachesize2[0]) self.assertEqual(cachesize[2], cachesize2[2]) self.assertEqual(125, int(100.0 * cachesize2[1] / cachesize[1])) self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) cachesize = (0, 2097152, 1) self.assertRaises(db.DBInvalidArgError, self.env.set_cachesize, *cachesize) cachesize3 = self.env.get_cachesize() self.assertEqual(cachesize2[0], cachesize3[0]) self.assertEqual(cachesize2[2], cachesize3[2]) self.assertTrue(cachesize2[1] <= cachesize3[1]) def test_set_cachesize_dbenv_db(self): d = db.DB(self.env) self.assertRaises(db.DBInvalidArgError, d.set_cachesize, 0, 1048576, 1) def test_setget_shm_key(self): shm_key = 137 self.env.set_shm_key(shm_key) self.assertEqual(shm_key, self.env.get_shm_key()) self.env.set_shm_key(shm_key + 1) self.assertEqual(shm_key + 1, self.env.get_shm_key()) self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) self.assertRaises(db.DBInvalidArgError, self.env.set_shm_key, shm_key) self.assertEqual(shm_key + 1, self.env.get_shm_key()) if db.version() >= (4, 4): def test_mutex_setget_max(self): v = self.env.mutex_get_max() v2 = v * 2 + 1 self.env.mutex_set_max(v2) self.assertEqual(v2, self.env.mutex_get_max()) self.env.mutex_set_max(v) self.assertEqual(v, self.env.mutex_get_max()) self.env.open(self.homeDir, db.DB_CREATE) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_max, v2) def test_mutex_setget_increment(self): v = self.env.mutex_get_increment() v2 = 127 self.env.mutex_set_increment(v2) self.assertEqual(v2, self.env.mutex_get_increment()) self.env.mutex_set_increment(v) self.assertEqual(v, self.env.mutex_get_increment()) self.env.open(self.homeDir, db.DB_CREATE) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_increment, v2) def test_mutex_setget_tas_spins(self): self.env.mutex_set_tas_spins(0) v = self.env.mutex_get_tas_spins() v2 = v * 2 + 1 self.env.mutex_set_tas_spins(v2) self.assertEqual(v2, self.env.mutex_get_tas_spins()) self.env.mutex_set_tas_spins(v) self.assertEqual(v, self.env.mutex_get_tas_spins()) self.env.open(self.homeDir, db.DB_CREATE) self.env.mutex_set_tas_spins(v2) def test_mutex_setget_align(self): v = self.env.mutex_get_align() v2 = 64 if v == 64: v2 = 128 self.env.mutex_set_align(v2) self.assertEqual(v2, self.env.mutex_get_align()) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_align, 0) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_align, 17) self.env.mutex_set_align(2 * v2) self.assertEqual(2 * v2, self.env.mutex_get_align()) self.env.open(self.homeDir, db.DB_CREATE) self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_align, v2)
class LockingTestCase(unittest.TestCase): import sys if sys.version_info[:3] < (2, 4, 0): def assertTrue(self, expr, msg=None): self.failUnless(expr,msg=msg) def setUp(self): self.homeDir = get_new_environment_path() self.env = db.DBEnv() self.env.open(self.homeDir, db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_CREATE) def tearDown(self): self.env.close() test_support.rmtree(self.homeDir) def test01_simple(self): if verbose: print '\n', '-=' * 30 print "Running %s.test01_simple..." % self.__class__.__name__ anID = self.env.lock_id() if verbose: print "locker ID: %s" % anID lock = self.env.lock_get(anID, "some locked thing", db.DB_LOCK_WRITE) if verbose: print "Aquired lock: %s" % lock self.env.lock_put(lock) if verbose: print "Released lock: %s" % lock self.env.lock_id_free(anID) def test02_threaded(self): if verbose: print '\n', '-=' * 30 print "Running %s.test02_threaded..." % self.__class__.__name__ threads = [] threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, args=(db.DB_LOCK_WRITE,))) for t in threads: import sys if sys.version_info[0] < 3 : t.setDaemon(True) else : t.daemon = True t.start() for t in threads: t.join() if db.version() >= (4, 2) : def test03_lock_timeout(self): self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT) self.assertEqual(self.env.get_timeout(db.DB_SET_LOCK_TIMEOUT), 0) self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT) self.assertEqual(self.env.get_timeout(db.DB_SET_TXN_TIMEOUT), 0) self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT) self.assertEqual(self.env.get_timeout(db.DB_SET_LOCK_TIMEOUT), 123456) self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT) self.assertEqual(self.env.get_timeout(db.DB_SET_TXN_TIMEOUT), 7890123) def test04_lock_timeout2(self): self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT) self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT) self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT) self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT) def deadlock_detection() : while not deadlock_detection.end : deadlock_detection.count = \ self.env.lock_detect(db.DB_LOCK_EXPIRE) if deadlock_detection.count : while not deadlock_detection.end : pass break time.sleep(0.01) deadlock_detection.end=False deadlock_detection.count=0 t=Thread(target=deadlock_detection) import sys if sys.version_info[0] < 3 : t.setDaemon(True) else : t.daemon = True t.start() self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT) anID = self.env.lock_id() anID2 = self.env.lock_id() self.assertNotEqual(anID, anID2) lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE) start_time=time.time() self.assertRaises(db.DBLockNotGrantedError, self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ) end_time=time.time() deadlock_detection.end=True self.assertTrue((end_time-start_time) >= 0.1) self.env.lock_put(lock) t.join() self.env.lock_id_free(anID) self.env.lock_id_free(anID2) if db.version() >= (4,6): self.assertTrue(deadlock_detection.count>0) def theThread(self, lockType): import sys if sys.version_info[0] < 3 : name = currentThread().getName() else : name = currentThread().name if lockType == db.DB_LOCK_WRITE: lt = "write" else: lt = "read" anID = self.env.lock_id() if verbose: print "%s: locker ID: %s" % (name, anID) for i in xrange(1000) : lock = self.env.lock_get(anID, "some locked thing", lockType) if verbose: print "%s: Aquired %s lock: %s" % (name, lt, lock) self.env.lock_put(lock) if verbose: print "%s: Released %s lock: %s" % (name, lt, lock) self.env.lock_id_free(anID)
def test01_basic_replication(self) : master_port = test_support.find_unused_port() self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) client_port = test_support.find_unused_port() self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321) self.assertEquals(self.dbenvMaster.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100123) self.assertEquals(self.dbenvClient.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEquals(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEquals(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEquals(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100345) self.assertEquals(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER); self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT); self.assertEquals(self.dbenvMaster.rep_get_nsites(),2) self.assertEquals(self.dbenvClient.rep_get_nsites(),2) self.assertEquals(self.dbenvMaster.rep_get_priority(),10) self.assertEquals(self.dbenvClient.rep_get_priority(),0) self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time()+10 while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : time.sleep(0.02) self.assertTrue(time.time()<timeout) d = self.dbenvMaster.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], client_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], master_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) if db.version() >= (4,6) : d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR); self.assertTrue("msgs_queued" in d) self.dbMaster=db.DB(self.dbenvMaster) txn=self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time,os.path timeout=time.time()+10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient=db.DB(self.dbenvClient) while True : txn=self.dbenvClient.txn_begin() try : self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError : txn.abort() self.dbClient.close() self.dbClient=db.DB(self.dbenvClient) continue txn.commit() break txn=self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout=time.time()+10 v=None while (time.time()<timeout) and (v==None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals("123", v) txn=self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout=time.time()+10 while (time.time()<timeout) and (v!=None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals(None, v)
def test03c_SimpleCursorGetReturnsNone2(self): # same test but raise exceptions instead of returning None if verbose: print '\n', '-=' * 30 print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \ self.__class__.__name__ old = self.d.set_get_returns_none(1) self.assertEqual(old, 2) old = self.d.set_get_returns_none(2) self.assertEqual(old, 1) self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0) if db.version() >= (4, 6): def test03d_SimpleCursorPriority(self) : c = self.d.cursor() c.set_priority(db.DB_PRIORITY_VERY_LOW) # Positional self.assertEqual(db.DB_PRIORITY_VERY_LOW, c.get_priority()) c.set_priority(priority=db.DB_PRIORITY_HIGH) # Keyword self.assertEqual(db.DB_PRIORITY_HIGH, c.get_priority()) c.close() #---------------------------------------- def test04_PartialGetAndPut(self): d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test04_PartialGetAndPut..." % \
def test01_basic_replication(self) : master_port = test_support.find_unused_port() self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) client_port = test_support.find_unused_port() self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321) self.assertEquals(self.dbenvMaster.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100123) self.assertEquals(self.dbenvClient.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEquals(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEquals(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEquals(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100345) self.assertEquals(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER); self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT); self.assertEquals(self.dbenvMaster.rep_get_nsites(),2) self.assertEquals(self.dbenvClient.rep_get_nsites(),2) self.assertEquals(self.dbenvMaster.rep_get_priority(),10) self.assertEquals(self.dbenvClient.rep_get_priority(),0) self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time()+10 while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : time.sleep(0.02) # this fails on Windows as self.client_startupdone never gets set # to True - see bug 3892. BUT - even though this assertion # fails on Windows the rest of the test passes - so to prove # that we let the rest of the test run. Sadly we can't # make use of raising TestSkipped() here (unittest still # reports it as an error), so we yell to stderr. import sys if sys.platform=="win32": print >> sys.stderr, \ "XXX - windows bsddb replication fails on windows and is skipped" print >> sys.stderr, "XXX - Please see issue #3892" else: self.assertTrue(time.time()<timeout) d = self.dbenvMaster.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], client_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], master_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) if db.version() >= (4,6) : d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR); self.assertTrue("msgs_queued" in d) self.dbMaster=db.DB(self.dbenvMaster) txn=self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time,os.path timeout=time.time()+10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient=db.DB(self.dbenvClient) while True : txn=self.dbenvClient.txn_begin() try : self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError : txn.abort() self.dbClient.close() self.dbClient=db.DB(self.dbenvClient) continue txn.commit() break txn=self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout=time.time()+10 v=None while (time.time()<timeout) and (v==None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals("123", v) txn=self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout=time.time()+10 while (time.time()<timeout) and (v!=None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals(None, v)
def test01_basic_replication(self) : master_port = test_support.find_unused_port() client_port = test_support.find_unused_port() if db.version() >= (5, 2) : self.site = self.dbenvMaster.repmgr_site("127.0.0.1", master_port) self.site.set_config(db.DB_GROUP_CREATOR, True) self.site.set_config(db.DB_LOCAL_SITE, True) self.site2 = self.dbenvMaster.repmgr_site("127.0.0.1", client_port) self.site3 = self.dbenvClient.repmgr_site("127.0.0.1", master_port) self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True) self.site4 = self.dbenvClient.repmgr_site("127.0.0.1", client_port) self.site4.set_config(db.DB_LOCAL_SITE, True) d = { db.DB_BOOTSTRAP_HELPER: [False, False, True, False], db.DB_GROUP_CREATOR: [True, False, False, False], db.DB_LEGACY: [False, False, False, False], db.DB_LOCAL_SITE: [True, False, False, True], db.DB_REPMGR_PEER: [False, False, False, False ], } for i, j in d.items() : for k, v in \ zip([self.site, self.site2, self.site3, self.site4], j) : if v : self.assertTrue(k.get_config(i)) else : self.assertFalse(k.get_config(i)) self.assertNotEqual(self.site.get_eid(), self.site2.get_eid()) self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid()) for i, j in zip([self.site, self.site2, self.site3, self.site4], \ [master_port, client_port, master_port, client_port]) : addr = i.get_address() self.assertEqual(addr, ("127.0.0.1", j)) for i in [self.site, self.site2] : self.assertEqual(i.get_address(), self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address()) for i in [self.site3, self.site4] : self.assertEqual(i.get_address(), self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address()) else : self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321) self.assertEqual(self.dbenvMaster.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100123) self.assertEqual(self.dbenvClient.rep_get_timeout( db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEqual(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEqual(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEqual(self.dbenvMaster.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100345) self.assertEqual(self.dbenvClient.rep_get_timeout( db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER); self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT); self.assertEqual(self.dbenvMaster.rep_get_nsites(),2) self.assertEqual(self.dbenvClient.rep_get_nsites(),2) self.assertEqual(self.dbenvMaster.rep_get_priority(),10) self.assertEqual(self.dbenvClient.rep_get_priority(),0) self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time()+10 while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : time.sleep(0.02) self.assertTrue(time.time()<timeout) d = self.dbenvMaster.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] # There is only one self.assertEqual(d[0], "127.0.0.1") self.assertEqual(d[1], client_port) self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \ (d[2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] # There is only one self.assertEqual(d[0], "127.0.0.1") self.assertEqual(d[1], master_port) self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \ (d[2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR); self.assertTrue("msgs_queued" in d) self.dbMaster=db.DB(self.dbenvMaster) txn=self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time,os.path timeout=time.time()+10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient=db.DB(self.dbenvClient) while True : txn=self.dbenvClient.txn_begin() try : self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError : txn.abort() self.dbClient.close() self.dbClient=db.DB(self.dbenvClient) continue txn.commit() break txn=self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout=time.time()+10 v=None while (time.time()<timeout) and (v is None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v is None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEqual("123", v) txn=self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout=time.time()+10 while (time.time()<timeout) and (v is not None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v is None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEqual(None, v)
def test_suite(): suite = unittest.TestSuite() if db.version() >= (4, 3): suite.addTest(unittest.makeSuite(DBSequenceTest)) return suite
class DBSequenceTest(unittest.TestCase): def setUp(self): self.int_32_max = 0x100000000 self.homeDir = get_new_environment_path() self.filename = "test" self.dbenv = db.DBEnv() self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL, 0666) self.d = db.DB(self.dbenv) self.d.open(self.filename, db.DB_BTREE, db.DB_CREATE, 0666) def tearDown(self): if hasattr(self, 'seq'): self.seq.close() del self.seq if hasattr(self, 'd'): self.d.close() del self.d if hasattr(self, 'dbenv'): self.dbenv.close() del self.dbenv test_support.rmtree(self.homeDir) def test_get(self): self.seq = db.DBSequence(self.d, flags=0) start_value = 10 * self.int_32_max self.assertEqual(0xA00000000, start_value) self.assertEqual(None, self.seq.initial_value(start_value)) self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE)) self.assertEqual(start_value, self.seq.get(5)) self.assertEqual(start_value + 5, self.seq.get()) def test_remove(self): self.seq = db.DBSequence(self.d, flags=0) self.assertEqual( None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(None, self.seq.remove(txn=None, flags=0)) del self.seq def test_get_key(self): self.seq = db.DBSequence(self.d, flags=0) key = 'foo' self.assertEqual(None, self.seq.open(key=key, txn=None, flags=db.DB_CREATE)) self.assertEqual(key, self.seq.get_key()) def test_get_dbp(self): self.seq = db.DBSequence(self.d, flags=0) self.assertEqual( None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(self.d, self.seq.get_dbp()) def test_cachesize(self): self.seq = db.DBSequence(self.d, flags=0) cashe_size = 10 self.assertEqual(None, self.seq.set_cachesize(cashe_size)) self.assertEqual( None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(cashe_size, self.seq.get_cachesize()) def test_flags(self): self.seq = db.DBSequence(self.d, flags=0) flag = db.DB_SEQ_WRAP self.assertEqual(None, self.seq.set_flags(flag)) self.assertEqual( None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(flag, self.seq.get_flags() & flag) def test_range(self): self.seq = db.DBSequence(self.d, flags=0) seq_range = (10 * self.int_32_max, 11 * self.int_32_max - 1) self.assertEqual(None, self.seq.set_range(seq_range)) self.seq.initial_value(seq_range[0]) self.assertEqual( None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(seq_range, self.seq.get_range()) def test_stat(self): self.seq = db.DBSequence(self.d, flags=0) self.assertEqual( None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) stat = self.seq.stat() for param in ('nowait', 'min', 'max', 'value', 'current', 'flags', 'cache_size', 'last_value', 'wait'): self.assertTrue(param in stat, "parameter %s isn't in stat info" % param) if db.version() >= (4, 7): # This code checks a crash solved in Berkeley DB 4.7 def test_stat_crash(self): d = db.DB() d.open(None, dbtype=db.DB_HASH, flags=db.DB_CREATE) # In RAM seq = db.DBSequence(d, flags=0) self.assertRaises(db.DBNotFoundError, seq.open, key='id', txn=None, flags=0) self.assertRaises(db.DBInvalidArgError, seq.stat) d.close() def test_64bits(self): # We don't use both extremes because they are problematic value_plus = (1L << 63) - 2 self.assertEqual(9223372036854775806L, value_plus) value_minus = (-1L << 63) + 1 # Two complement self.assertEqual(-9223372036854775807L, value_minus) self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.initial_value(value_plus - 1)) self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE)) self.assertEqual(value_plus - 1, self.seq.get(1)) self.assertEqual(value_plus, self.seq.get(1)) self.seq.remove(txn=None, flags=0) self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.initial_value(value_minus)) self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE)) self.assertEqual(value_minus, self.seq.get(1)) self.assertEqual(value_minus + 1, self.seq.get(1)) def test_multiple_close(self): self.seq = db.DBSequence(self.d) self.seq.close() # You can close a Sequence multiple times self.seq.close() self.seq.close()
class DBBaseReplication(DBReplicationManager): def setUp(self) : DBReplicationManager.setUp(self) def confirmed_master(a,b,c) : if (b == db.DB_EVENT_REP_MASTER) or (b == db.DB_EVENT_REP_ELECTED) : self.confirmed_master = True def client_startupdone(a,b,c) : if b == db.DB_EVENT_REP_STARTUPDONE : self.client_startupdone = True self.dbenvMaster.set_event_notify(confirmed_master) self.dbenvClient.set_event_notify(client_startupdone) import Queue self.m2c = Queue.Queue() self.c2m = Queue.Queue() # There are only two nodes, so we don't need to # do any routing decision def m2c(dbenv, control, rec, lsnp, envid, flags) : self.m2c.put((control, rec)) def c2m(dbenv, control, rec, lsnp, envid, flags) : self.c2m.put((control, rec)) self.dbenvMaster.rep_set_transport(13,m2c) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_transport(3,c2m) self.dbenvClient.rep_set_priority(0) self.assertEquals(self.dbenvMaster.rep_get_priority(),10) self.assertEquals(self.dbenvClient.rep_get_priority(),0) #self.dbenvMaster.set_verbose(db.DB_VERB_REPLICATION, True) #self.dbenvMaster.set_verbose(db.DB_VERB_FILEOPS_ALL, True) #self.dbenvClient.set_verbose(db.DB_VERB_REPLICATION, True) #self.dbenvClient.set_verbose(db.DB_VERB_FILEOPS_ALL, True) def thread_master() : return self.thread_do(self.dbenvMaster, self.c2m, 3, self.master_doing_election, True) def thread_client() : return self.thread_do(self.dbenvClient, self.m2c, 13, self.client_doing_election, False) from threading import Thread t_m=Thread(target=thread_master) t_c=Thread(target=thread_client) import sys if sys.version_info[0] < 3 : t_m.setDaemon(True) t_c.setDaemon(True) else : t_m.daemon = True t_c.daemon = True self.t_m = t_m self.t_c = t_c self.dbMaster = self.dbClient = None self.master_doing_election=[False] self.client_doing_election=[False] def tearDown(self): if self.dbClient : self.dbClient.close() if self.dbMaster : self.dbMaster.close() self.m2c.put(None) self.c2m.put(None) self.t_m.join() self.t_c.join() self.dbenvClient.close() self.dbenvMaster.close() test_support.rmtree(self.homeDirClient) test_support.rmtree(self.homeDirMaster) def basic_rep_threading(self) : self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER) self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT) def thread_do(env, q, envid, election_status, must_be_master) : while True : v=q.get() if v == None : return env.rep_process_message(v[0], v[1], envid) self.thread_do = thread_do self.t_m.start() self.t_c.start() def test01_basic_replication(self) : self.basic_rep_threading() # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time()+10 while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.dbMaster=db.DB(self.dbenvMaster) txn=self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time,os.path timeout=time.time()+10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient=db.DB(self.dbenvClient) while True : txn=self.dbenvClient.txn_begin() try : self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError : txn.abort() self.dbClient.close() self.dbClient=db.DB(self.dbenvClient) continue txn.commit() break d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR); self.assertTrue("master_changes" in d) txn=self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout=time.time()+10 v=None while (time.time()<timeout) and (v==None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals("123", v) txn=self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout=time.time()+10 while (time.time()<timeout) and (v!=None) : txn=self.dbenvClient.txn_begin() v=self.dbClient.get("ABC", txn=txn) txn.commit() if v==None : time.sleep(0.02) self.assertTrue(time.time()<timeout) self.assertEquals(None, v) if db.version() >= (4,7) : def test02_test_request(self) : self.basic_rep_threading() (minimum, maximum) = self.dbenvClient.rep_get_request() self.dbenvClient.rep_set_request(minimum-1, maximum+1) self.assertEqual(self.dbenvClient.rep_get_request(), (minimum-1, maximum+1)) if db.version() >= (4,6) : def test03_master_election(self) : # Get ready to hold an election #self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER) self.dbenvMaster.rep_start(flags=db.DB_REP_CLIENT) self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT) def thread_do(env, q, envid, election_status, must_be_master) : while True : v=q.get() if v == None : return r = env.rep_process_message(v[0],v[1],envid) if must_be_master and self.confirmed_master : self.dbenvMaster.rep_start(flags = db.DB_REP_MASTER) must_be_master = False if r[0] == db.DB_REP_HOLDELECTION : def elect() : while True : try : env.rep_elect(2, 1) election_status[0] = False break except db.DBRepUnavailError : pass if not election_status[0] and not self.confirmed_master : from threading import Thread election_status[0] = True t=Thread(target=elect) import sys if sys.version_info[0] < 3 : t.setDaemon(True) else : t.daemon = True t.start() self.thread_do = thread_do self.t_m.start() self.t_c.start() self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000) self.client_doing_election[0] = True while True : try : self.dbenvClient.rep_elect(2, 1) self.client_doing_election[0] = False break except db.DBRepUnavailError : pass self.assertTrue(self.confirmed_master) if db.version() >= (4,7) : def test04_test_clockskew(self) : fast, slow = 1234, 1230 self.dbenvMaster.rep_set_clockskew(fast, slow) self.assertEqual((fast, slow), self.dbenvMaster.rep_get_clockskew()) self.basic_rep_threading()
def _destroy_env(self): if self.nosync or db.version()[:2] == (4, 6): self.dbenv.log_flush() self.db.close() self.dbenv.close()
class BasicTestCase(unittest.TestCase): dbtype = db.DB_UNKNOWN cachesize = (0, 1048576, 1) dbopenflags = 0 dbsetflags = 0 dbmode = 432 dbname = None useEnv = 0 envflags = 0 envsetflags = 0 _numKeys = 1002 def setUp(self): if self.useEnv: self.homeDir = get_new_environment_path() try: self.env = db.DBEnv() self.env.set_lg_max(1048576) self.env.set_tx_max(30) self._t = int(time.time()) self.env.set_tx_timestamp(self._t) self.env.set_flags(self.envsetflags, 1) self.env.open(self.homeDir, self.envflags | db.DB_CREATE) self.filename = 'test' except: test_support.rmtree(self.homeDir) raise else: self.env = None self.filename = get_new_database_path() self.d = db.DB(self.env) if not self.useEnv: self.d.set_cachesize(*self.cachesize) cachesize = self.d.get_cachesize() self.assertEqual(cachesize[0], self.cachesize[0]) self.assertEqual(cachesize[2], self.cachesize[2]) self.assertEqual(125, int(100.0 * cachesize[1] / self.cachesize[1])) self.d.set_flags(self.dbsetflags) if self.dbname: self.d.open(self.filename, self.dbname, self.dbtype, self.dbopenflags | db.DB_CREATE, self.dbmode) else: self.d.open(self.filename, mode=self.dbmode, dbtype=self.dbtype, flags=self.dbopenflags | db.DB_CREATE) if not self.useEnv: self.assertRaises(db.DBInvalidArgError, self.d.set_cachesize, *self.cachesize) self.populateDB() return def tearDown(self): self.d.close() if self.env is not None: self.env.close() test_support.rmtree(self.homeDir) else: os.remove(self.filename) return def populateDB(self, _txn=None): d = self.d for x in range(self._numKeys // 2): key = '%04d' % (self._numKeys - x) data = self.makeData(key) d.put(key, data, _txn) d.put('empty value', '', _txn) for x in range(self._numKeys // 2 - 1): key = '%04d' % x data = self.makeData(key) d.put(key, data, _txn) if _txn: _txn.commit() num = len(d) if verbose: print 'created %d records' % num def makeData(self, key): return DASH.join([key] * 5) def test01_GetsAndPuts(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test01_GetsAndPuts...' % self.__class__.__name__ for key in ['0001', '0100', '0400', '0700', '0999']: data = d.get(key) if verbose: print data self.assertEqual(d.get('0321'), '0321-0321-0321-0321-0321') self.assertEqual(d.get('abcd'), None) try: d.delete('abcd') except db.DBNotFoundError as val: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_NOTFOUND) else: self.assertEqual(val.args[0], db.DB_NOTFOUND) if verbose: print val else: self.fail('expected exception') d.put('abcd', 'a new record') self.assertEqual(d.get('abcd'), 'a new record') d.put('abcd', 'same key') if self.dbsetflags & db.DB_DUP: self.assertEqual(d.get('abcd'), 'a new record') else: self.assertEqual(d.get('abcd'), 'same key') try: d.put('abcd', 'this should fail', flags=db.DB_NOOVERWRITE) except db.DBKeyExistError as val: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_KEYEXIST) else: self.assertEqual(val.args[0], db.DB_KEYEXIST) if verbose: print val else: self.fail('expected exception') if self.dbsetflags & db.DB_DUP: self.assertEqual(d.get('abcd'), 'a new record') else: self.assertEqual(d.get('abcd'), 'same key') d.sync() d.close() del d self.d = db.DB(self.env) if self.dbname: self.d.open(self.filename, self.dbname) else: self.d.open(self.filename) d = self.d self.assertEqual(d.get('0321'), '0321-0321-0321-0321-0321') if self.dbsetflags & db.DB_DUP: self.assertEqual(d.get('abcd'), 'a new record') else: self.assertEqual(d.get('abcd'), 'same key') rec = d.get_both('0555', '0555-0555-0555-0555-0555') if verbose: print rec self.assertEqual(d.get_both('0555', 'bad data'), None) data = d.get('bad key', 'bad data') self.assertEqual(data, 'bad data') data = d.get('bad key', self) self.assertEqual(data, self) s = d.stat() self.assertEqual(type(s), type({})) if verbose: print 'd.stat() returned this dictionary:' pprint(s) return def test02_DictionaryMethods(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test02_DictionaryMethods...' % self.__class__.__name__ for key in ['0002', '0101', '0401', '0701', '0998']: data = d[key] self.assertEqual(data, self.makeData(key)) if verbose: print data self.assertEqual(len(d), self._numKeys) keys = d.keys() self.assertEqual(len(keys), self._numKeys) self.assertEqual(type(keys), type([])) d['new record'] = 'a new record' self.assertEqual(len(d), self._numKeys + 1) keys = d.keys() self.assertEqual(len(keys), self._numKeys + 1) d['new record'] = 'a replacement record' self.assertEqual(len(d), self._numKeys + 1) keys = d.keys() self.assertEqual(len(keys), self._numKeys + 1) if verbose: print 'the first 10 keys are:' pprint(keys[:10]) self.assertEqual(d['new record'], 'a replacement record') self.assertEqual(d.has_key('0001', None), 1) self.assertEqual(d.has_key('spam', txn=None), 0) items = d.items() self.assertEqual(len(items), self._numKeys + 1) self.assertEqual(type(items), type([])) self.assertEqual(type(items[0]), type(())) self.assertEqual(len(items[0]), 2) if verbose: print 'the first 10 items are:' pprint(items[:10]) values = d.values() self.assertEqual(len(values), self._numKeys + 1) self.assertEqual(type(values), type([])) if verbose: print 'the first 10 values are:' pprint(values[:10]) return def test02b_SequenceMethods(self): d = self.d for key in ['0002', '0101', '0401', '0701', '0998']: data = d[key] self.assertEqual(data, self.makeData(key)) if verbose: print data self.assertTrue(hasattr(d, '__contains__')) self.assertTrue('0401' in d) self.assertFalse('1234' in d) def test03_SimpleCursorStuff(self, get_raises_error=0, set_raises_error=0): if verbose: print '\n', '-=' * 30 print 'Running %s.test03_SimpleCursorStuff (get_error %s, set_error %s)...' % ( self.__class__.__name__, get_raises_error, set_raises_error) if self.env and self.dbopenflags & db.DB_AUTO_COMMIT: txn = self.env.txn_begin() else: txn = None c = self.d.cursor(txn=txn) rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec try: rec = c.next() except db.DBNotFoundError as val: if get_raises_error: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_NOTFOUND) else: self.assertEqual(val.args[0], db.DB_NOTFOUND) if verbose: print val rec = None else: self.fail('unexpected DBNotFoundError') self.assertEqual( c.get_current_size(), len(c.current()[1]), '%s != len(%r)' % (c.get_current_size(), c.current()[1])) self.assertEqual(count, self._numKeys) rec = c.last() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec try: rec = c.prev() except db.DBNotFoundError as val: if get_raises_error: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_NOTFOUND) else: self.assertEqual(val.args[0], db.DB_NOTFOUND) if verbose: print val rec = None else: self.fail('unexpected DBNotFoundError') self.assertEqual(count, self._numKeys) rec = c.set('0505') rec2 = c.current() self.assertEqual(rec, rec2) self.assertEqual(rec[0], '0505') self.assertEqual(rec[1], self.makeData('0505')) self.assertEqual(c.get_current_size(), len(rec[1])) rec = c.set('empty value') self.assertEqual(rec[1], '') self.assertEqual(c.get_current_size(), 0) try: n = c.set('bad key') except db.DBNotFoundError as val: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_NOTFOUND) else: self.assertEqual(val.args[0], db.DB_NOTFOUND) if verbose: print val else: if set_raises_error: self.fail('expected exception') if n is not None: self.fail('expected None: %r' % (n, )) rec = c.get_both('0404', self.makeData('0404')) self.assertEqual(rec, ('0404', self.makeData('0404'))) try: n = c.get_both('0404', 'bad data') except db.DBNotFoundError as val: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_NOTFOUND) else: self.assertEqual(val.args[0], db.DB_NOTFOUND) if verbose: print val else: if get_raises_error: self.fail('expected exception') if n is not None: self.fail('expected None: %r' % (n, )) if self.d.get_type() == db.DB_BTREE: rec = c.set_range('011') if verbose: print "searched for '011', found: ", rec rec = c.set_range('011', dlen=0, doff=0) if verbose: print "searched (partial) for '011', found: ", rec if rec[1] != '': self.fail('expected empty data portion') ev = c.set_range('empty value') if verbose: print "search for 'empty value' returned", ev if ev[1] != '': self.fail('empty value lookup failed') c.set('0499') c.delete() try: rec = c.current() except db.DBKeyEmptyError as val: if get_raises_error: if sys.version_info < (2, 6): self.assertEqual(val[0], db.DB_KEYEMPTY) else: self.assertEqual(val.args[0], db.DB_KEYEMPTY) if verbose: print val else: self.fail('unexpected DBKeyEmptyError') else: if get_raises_error: self.fail('DBKeyEmptyError exception expected') c.next() c2 = c.dup(db.DB_POSITION) self.assertEqual(c.current(), c2.current()) c2.put('', 'a new value', db.DB_CURRENT) self.assertEqual(c.current(), c2.current()) self.assertEqual(c.current()[1], 'a new value') c2.put('', 'er', db.DB_CURRENT, dlen=0, doff=5) self.assertEqual(c2.current()[1], 'a newer value') c.close() c2.close() if txn: txn.commit() methods_to_test = { 'current': (), 'delete': (), 'dup': (db.DB_POSITION, ), 'first': (), 'get': (0, ), 'next': (), 'prev': (), 'last': (), 'put': ('', 'spam', db.DB_CURRENT), 'set': ('0505', ) } for method, args in methods_to_test.items(): try: if verbose: print "attempting to use a closed cursor's %s method" % method getattr(c, method)(*args) except db.DBError as val: if sys.version_info < (2, 6): self.assertEqual(val[0], 0) else: self.assertEqual(val.args[0], 0) if verbose: print val else: self.fail( "no exception raised when using a buggy cursor's%s method" % method) oldcursor = self.d.cursor(txn=txn) self.d.close() del oldcursor return def test03b_SimpleCursorWithoutGetReturnsNone0(self): if verbose: print '\n', '-=' * 30 print 'Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone...' % self.__class__.__name__ old = self.d.set_get_returns_none(0) self.assertEqual(old, 2) self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1) def test03b_SimpleCursorWithGetReturnsNone1(self): if verbose: print '\n', '-=' * 30 print 'Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone...' % self.__class__.__name__ old = self.d.set_get_returns_none(1) self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=1) def test03c_SimpleCursorGetReturnsNone2(self): if verbose: print '\n', '-=' * 30 print 'Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone...' % self.__class__.__name__ old = self.d.set_get_returns_none(1) self.assertEqual(old, 2) old = self.d.set_get_returns_none(2) self.assertEqual(old, 1) self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0) if db.version() >= (4, 6): def test03d_SimpleCursorPriority(self): c = self.d.cursor() c.set_priority(db.DB_PRIORITY_VERY_LOW) self.assertEqual(db.DB_PRIORITY_VERY_LOW, c.get_priority()) c.set_priority(priority=db.DB_PRIORITY_HIGH) self.assertEqual(db.DB_PRIORITY_HIGH, c.get_priority()) c.close() def test04_PartialGetAndPut(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test04_PartialGetAndPut...' % self.__class__.__name__ key = 'partialTest' data = '1' * 1000 + '2' * 1000 d.put(key, data) self.assertEqual(d.get(key), data) self.assertEqual(d.get(key, dlen=20, doff=990), '1111111111' + '2222222222') d.put('partialtest2', '1' * 30000 + 'robin') self.assertEqual(d.get('partialtest2', dlen=5, doff=30000), 'robin') if self.dbsetflags != db.DB_DUP: d.put(key, '0000', dlen=2000, doff=0) self.assertEqual(d.get(key), '0000') d.put(key, '1111', dlen=1, doff=2) self.assertEqual(d.get(key), '0011110') def test05_GetSize(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test05_GetSize...' % self.__class__.__name__ for i in range(1, 50000, 500): key = 'size%s' % i d.put(key, '1' * i) self.assertEqual(d.get_size(key), i) def test06_Truncate(self): d = self.d if verbose: print '\n', '-=' * 30 print 'Running %s.test06_Truncate...' % self.__class__.__name__ d.put('abcde', 'ABCDE') num = d.truncate() self.assertTrue(num >= 1, 'truncate returned <= 0 on non-empty database') num = d.truncate() self.assertEqual( num, 0, 'truncate on empty DB returned nonzero (%r)' % (num, )) def test07_verify(self): self.d.close() d = db.DB(self.env) d.verify(self.filename) if db.version() >= (4, 6): def test08_exists(self): self.d.put('abcde', 'ABCDE') self.assertTrue( self.d.exists('abcde') == True, 'DB->exists() returns wrong value') self.assertTrue( self.d.exists('x') == False, 'DB->exists() returns wrong value') if db.version() >= (4, 7): def test_compact(self): d = self.d self.assertEqual(0, d.compact(flags=db.DB_FREELIST_ONLY)) self.assertEqual(0, d.compact(flags=db.DB_FREELIST_ONLY)) d.put('abcde', 'ABCDE') d.put('bcde', 'BCDE') d.put('abc', 'ABC') d.put('monty', 'python') d.delete('abc') d.delete('bcde') d.compact(start='abcde', stop='monty', txn=None, compact_fillpercent=42, compact_pages=1, compact_timeout=50000000, flags=db.DB_FREELIST_ONLY | db.DB_FREE_SPACE) return
class BasicTransactionTestCase(BasicTestCase): import sys if sys.version_info < (2, 4): def assertTrue(self, expr, msg=None): return self.failUnless(expr, msg=msg) if (sys.version_info < (2, 7)) or ((sys.version_info >= (3, 0)) and (sys.version_info < (3, 2))): def assertIn(self, a, b, msg=None): return self.assertTrue(a in b, msg=msg) dbopenflags = db.DB_THREAD | db.DB_AUTO_COMMIT useEnv = 1 envflags = (db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_TXN) envsetflags = db.DB_AUTO_COMMIT def tearDown(self): self.txn.commit() BasicTestCase.tearDown(self) def populateDB(self): txn = self.env.txn_begin() BasicTestCase.populateDB(self, _txn=txn) self.txn = self.env.txn_begin() def test06_Transactions(self): d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test06_Transactions..." % self.__class__.__name__ self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) self.assertEqual(d.get('new rec', txn=self.txn), 'this is a new record') self.txn.abort() self.assertEqual(d.get('new rec'), None) self.txn = self.env.txn_begin() self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) self.assertEqual(d.get('new rec', txn=self.txn), 'this is a new record') self.txn.commit() self.assertEqual(d.get('new rec'), 'this is a new record') self.txn = self.env.txn_begin() c = d.cursor(self.txn) rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec rec = c.next() self.assertEqual(count, self._numKeys + 1) c.close() # Cursors *MUST* be closed before commit! self.txn.commit() # flush pending updates self.env.txn_checkpoint(0, 0, 0) statDict = self.env.log_stat(0) self.assertIn('magic', statDict) self.assertIn('version', statDict) self.assertIn('cur_file', statDict) self.assertIn('region_nowait', statDict) # must have at least one log file present: logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG) self.assertNotEqual(logs, None) for log in logs: if verbose: print 'log file: ' + log if db.version() >= (4, 2): logs = self.env.log_archive(db.DB_ARCH_REMOVE) self.assertTrue(not logs) self.txn = self.env.txn_begin() #---------------------------------------- if db.version() >= (4, 6): def test08_exists(self): txn = self.env.txn_begin() self.d.put("abcde", "ABCDE", txn=txn) txn.commit() txn = self.env.txn_begin() self.assertTrue( self.d.exists("abcde", txn=txn) == True, "DB->exists() returns wrong value") self.assertTrue( self.d.exists("x", txn=txn) == False, "DB->exists() returns wrong value") txn.abort() #---------------------------------------- def test09_TxnTruncate(self): d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test09_TxnTruncate..." % self.__class__.__name__ d.put("abcde", "ABCDE") txn = self.env.txn_begin() num = d.truncate(txn) self.assertTrue(num >= 1, "truncate returned <= 0 on non-empty database") num = d.truncate(txn) self.assertEqual( num, 0, "truncate on empty DB returned nonzero (%r)" % (num, )) txn.commit() #---------------------------------------- def test10_TxnLateUse(self): txn = self.env.txn_begin() txn.abort() try: txn.abort() except db.DBError, e: pass else:
def test01_basic_replication(self): master_port = test_support.find_unused_port() client_port = test_support.find_unused_port() if db.version() >= (5, 2): self.site = self.dbenvMaster.repmgr_site('127.0.0.1', master_port) self.site.set_config(db.DB_GROUP_CREATOR, True) self.site.set_config(db.DB_LOCAL_SITE, True) self.site2 = self.dbenvMaster.repmgr_site('127.0.0.1', client_port) self.site3 = self.dbenvClient.repmgr_site('127.0.0.1', master_port) self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True) self.site4 = self.dbenvClient.repmgr_site('127.0.0.1', client_port) self.site4.set_config(db.DB_LOCAL_SITE, True) d = {db.DB_BOOTSTRAP_HELPER: [False, False, True, False], db.DB_GROUP_CREATOR: [True, False, False, False], db.DB_LEGACY: [False, False, False, False], db.DB_LOCAL_SITE: [True, False, False, True], db.DB_REPMGR_PEER: [False, False, False, False]} for i, j in d.items(): for k, v in zip([self.site, self.site2, self.site3, self.site4], j): if v: self.assertTrue(k.get_config(i)) else: self.assertFalse(k.get_config(i)) self.assertNotEqual(self.site.get_eid(), self.site2.get_eid()) self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid()) for i, j in zip([self.site, self.site2, self.site3, self.site4], [master_port, client_port, master_port, client_port]): addr = i.get_address() self.assertEqual(addr, ('127.0.0.1', j)) for i in [self.site, self.site2]: self.assertEqual(i.get_address(), self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address()) for i in [self.site3, self.site4]: self.assertEqual(i.get_address(), self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address()) else: self.dbenvMaster.repmgr_set_local_site('127.0.0.1', master_port) self.dbenvClient.repmgr_set_local_site('127.0.0.1', client_port) self.dbenvMaster.repmgr_add_remote_site('127.0.0.1', client_port) self.dbenvClient.repmgr_add_remote_site('127.0.0.1', master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321) self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123) self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345) self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER) self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT) self.assertEqual(self.dbenvMaster.rep_get_nsites(), 2) self.assertEqual(self.dbenvClient.rep_get_nsites(), 2) self.assertEqual(self.dbenvMaster.rep_get_priority(), 10) self.assertEqual(self.dbenvClient.rep_get_priority(), 0) self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) import time timeout = time.time() + 60 while time.time() < timeout and not (self.confirmed_master and self.client_startupdone): time.sleep(0.02) if time.time() > timeout: import sys print >> sys.stderr, 'XXX: timeout happened beforestartup was confirmed - see issue 3892' startup_timeout = True d = self.dbenvMaster.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] self.assertEqual(d[0], '127.0.0.1') self.assertEqual(d[1], client_port) self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED or d[2] == db.DB_REPMGR_DISCONNECTED) d = self.dbenvClient.repmgr_site_list() self.assertEqual(len(d), 1) d = d.values()[0] self.assertEqual(d[0], '127.0.0.1') self.assertEqual(d[1], master_port) self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED or d[2] == db.DB_REPMGR_DISCONNECTED) if db.version() >= (4, 6): d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR) self.assertTrue('msgs_queued' in d) self.dbMaster = db.DB(self.dbenvMaster) txn = self.dbenvMaster.txn_begin() self.dbMaster.open('test', db.DB_HASH, db.DB_CREATE, 438, txn=txn) txn.commit() import time, os.path timeout = time.time() + 10 while time.time() < timeout and not os.path.exists(os.path.join(self.homeDirClient, 'test')): time.sleep(0.01) self.dbClient = db.DB(self.dbenvClient) while True: txn = self.dbenvClient.txn_begin() try: self.dbClient.open('test', db.DB_HASH, flags=db.DB_RDONLY, mode=438, txn=txn) except db.DBRepHandleDeadError: txn.abort() self.dbClient.close() self.dbClient = db.DB(self.dbenvClient) continue txn.commit() break txn = self.dbenvMaster.txn_begin() self.dbMaster.put('ABC', '123', txn=txn) txn.commit() import time timeout = time.time() + 10 v = None while time.time() < timeout and v is None: txn = self.dbenvClient.txn_begin() v = self.dbClient.get('ABC', txn=txn) txn.commit() if v is None: time.sleep(0.02) if time.time() >= timeout and startup_timeout: self.skipTest('replication test skipped due to random failure, see issue 3892') self.assertTrue(time.time() < timeout) self.assertEqual('123', v) txn = self.dbenvMaster.txn_begin() self.dbMaster.delete('ABC', txn=txn) txn.commit() timeout = time.time() + 10 while time.time() < timeout and v is not None: txn = self.dbenvClient.txn_begin() v = self.dbClient.get('ABC', txn=txn) txn.commit() if v is None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEqual(None, v) return
class DBEnvClosedEarlyCrash(unittest.TestCase): def setUp(self): self.homeDir = get_new_environment_path() self.filename = "test" def tearDown(self): test_support.rmtree(self.homeDir) def test01_close_dbenv_before_db(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) d = db.DB(dbenv) d2 = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) self.assertRaises(db.DBNoSuchFileError, d2.open, self.filename + "2", db.DB_BTREE, db.DB_THREAD, 0666) d.put("test", "this is a test") self.assertEqual(d.get("test"), "this is a test", "put!=get") dbenv.close() # This "close" should close the child db handle also self.assertRaises(db.DBError, d.get, "test") def test02_close_dbenv_before_dbcursor(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) d = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) d.put("test", "this is a test") d.put("test2", "another test") d.put("test3", "another one") self.assertEqual(d.get("test"), "this is a test", "put!=get") c = d.cursor() c.first() c.next() d.close() # This "close" should close the child db handle also # db.close should close the child cursor self.assertRaises(db.DBError, c.next) d = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) c = d.cursor() c.first() c.next() dbenv.close() # The "close" should close the child db handle also, with cursors self.assertRaises(db.DBError, c.next) def test03_close_db_before_dbcursor_without_env(self): import os.path path = os.path.join(self.homeDir, self.filename) d = db.DB() d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) d.put("test", "this is a test") d.put("test2", "another test") d.put("test3", "another one") self.assertEqual(d.get("test"), "this is a test", "put!=get") c = d.cursor() c.first() c.next() d.close() # The "close" should close the child db handle also self.assertRaises(db.DBError, c.next) def test04_close_massive(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) dbs = [db.DB(dbenv) for i in xrange(16)] cursors = [] for i in dbs: i.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) dbs[10].put("test", "this is a test") dbs[10].put("test2", "another test") dbs[10].put("test3", "another one") self.assertEqual(dbs[4].get("test"), "this is a test", "put!=get") for i in dbs: cursors.extend([i.cursor() for j in xrange(32)]) for i in dbs[::3]: i.close() for i in cursors[::3]: i.close() # Check for missing exception in DB! (after DB close) self.assertRaises(db.DBError, dbs[9].get, "test") # Check for missing exception in DBCursor! (after DB close) self.assertRaises(db.DBError, cursors[101].first) cursors[80].first() cursors[80].next() dbenv.close() # This "close" should close the child db handle also # Check for missing exception! (after DBEnv close) self.assertRaises(db.DBError, cursors[80].next) def test05_close_dbenv_delete_db_success(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL, 0666) d = db.DB(dbenv) d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) dbenv.close() # This "close" should close the child db handle also del d try: import gc except ImportError: gc = None if gc: # force d.__del__ [DB_dealloc] to be called gc.collect() def test06_close_txn_before_dup_cursor(self): dbenv = db.DBEnv() dbenv.open( self.homeDir, db.DB_INIT_TXN | db.DB_INIT_MPOOL | db.DB_INIT_LOG | db.DB_CREATE) d = db.DB(dbenv) txn = dbenv.txn_begin() if db.version() < (4, 1): d.open(self.filename, dbtype=db.DB_HASH, flags=db.DB_CREATE) else: d.open(self.filename, dbtype=db.DB_HASH, flags=db.DB_CREATE, txn=txn) d.put("XXX", "yyy", txn=txn) txn.commit() txn = dbenv.txn_begin() c1 = d.cursor(txn) c2 = c1.dup() self.assertEquals(("XXX", "yyy"), c1.first()) import warnings # Not interested in warnings about implicit close. with warnings.catch_warnings(): warnings.simplefilter("ignore") txn.commit() self.assertRaises(db.DBCursorClosedError, c2.first) if db.version() > (4, 3, 0): def test07_close_db_before_sequence(self): import os.path path = os.path.join(self.homeDir, self.filename) d = db.DB() d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) dbs = db.DBSequence(d) d.close() # This "close" should close the child DBSequence also dbs.close() # If not closed, core dump (in Berkeley DB 4.6.*)
def test_suite(): suite = unittest.TestSuite() if db.version() >= (4,3): suite.addTest(unittest.makeSuite(DBSequenceTest)) return suite
class DBSequenceTest(unittest.TestCase): def setUp(self): self.int_32_max = 4294967296L self.homeDir = get_new_environment_path() self.filename = 'test' self.dbenv = db.DBEnv() self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL, 438) self.d = db.DB(self.dbenv) self.d.open(self.filename, db.DB_BTREE, db.DB_CREATE, 438) def tearDown(self): if hasattr(self, 'seq'): self.seq.close() del self.seq if hasattr(self, 'd'): self.d.close() del self.d if hasattr(self, 'dbenv'): self.dbenv.close() del self.dbenv test_support.rmtree(self.homeDir) def test_get(self): self.seq = db.DBSequence(self.d, flags=0) start_value = 10 * self.int_32_max self.assertEqual(42949672960L, start_value) self.assertEqual(None, self.seq.initial_value(start_value)) self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE)) self.assertEqual(start_value, self.seq.get(5)) self.assertEqual(start_value + 5, self.seq.get()) return def test_remove(self): self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(None, self.seq.remove(txn=None, flags=0)) del self.seq return def test_get_key(self): self.seq = db.DBSequence(self.d, flags=0) key = 'foo' self.assertEqual(None, self.seq.open(key=key, txn=None, flags=db.DB_CREATE)) self.assertEqual(key, self.seq.get_key()) return def test_get_dbp(self): self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(self.d, self.seq.get_dbp()) return def test_cachesize(self): self.seq = db.DBSequence(self.d, flags=0) cashe_size = 10 self.assertEqual(None, self.seq.set_cachesize(cashe_size)) self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(cashe_size, self.seq.get_cachesize()) return def test_flags(self): self.seq = db.DBSequence(self.d, flags=0) flag = db.DB_SEQ_WRAP self.assertEqual(None, self.seq.set_flags(flag)) self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(flag, self.seq.get_flags() & flag) return def test_range(self): self.seq = db.DBSequence(self.d, flags=0) seq_range = (10 * self.int_32_max, 11 * self.int_32_max - 1) self.assertEqual(None, self.seq.set_range(seq_range)) self.seq.initial_value(seq_range[0]) self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) self.assertEqual(seq_range, self.seq.get_range()) return def test_stat(self): self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE)) stat = self.seq.stat() for param in ('nowait', 'min', 'max', 'value', 'current', 'flags', 'cache_size', 'last_value', 'wait'): self.assertTrue(param in stat, "parameter %s isn't in stat info" % param) return if db.version() >= (4, 7): def test_stat_crash(self): d = db.DB() d.open(None, dbtype=db.DB_HASH, flags=db.DB_CREATE) seq = db.DBSequence(d, flags=0) self.assertRaises(db.DBNotFoundError, seq.open, key='id', txn=None, flags=0) self.assertRaises(db.DBInvalidArgError, seq.stat) d.close() return def test_64bits(self): value_plus = 9223372036854775806L self.assertEqual(9223372036854775806L, value_plus) value_minus = -9223372036854775807L self.assertEqual(-9223372036854775807L, value_minus) self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.initial_value(value_plus - 1)) self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE)) self.assertEqual(value_plus - 1, self.seq.get(1)) self.assertEqual(value_plus, self.seq.get(1)) self.seq.remove(txn=None, flags=0) self.seq = db.DBSequence(self.d, flags=0) self.assertEqual(None, self.seq.initial_value(value_minus)) self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE)) self.assertEqual(value_minus, self.seq.get(1)) self.assertEqual(value_minus + 1, self.seq.get(1)) return def test_multiple_close(self): self.seq = db.DBSequence(self.d) self.seq.close() self.seq.close() self.seq.close()
def _destroy_env(self): if self.nosync or (db.version()[:2] == (4,6)): # Known bug self.dbenv.log_flush() self.db.close() self.dbenv.close()
def test01_basic_replication(self): master_port = test_support.find_unused_port() self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port) client_port = test_support.find_unused_port() self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port) self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port) self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port) self.dbenvMaster.rep_set_nsites(2) self.dbenvClient.rep_set_nsites(2) self.dbenvMaster.rep_set_priority(10) self.dbenvClient.rep_set_priority(0) self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123) self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321) self.assertEquals( self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123) self.assertEquals( self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432) self.assertEquals( self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234) self.assertEquals( self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432) self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345) self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543) self.assertEquals( self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345) self.assertEquals( self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543) self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER) self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT) self.assertEquals(self.dbenvMaster.rep_get_nsites(), 2) self.assertEquals(self.dbenvClient.rep_get_nsites(), 2) self.assertEquals(self.dbenvMaster.rep_get_priority(), 10) self.assertEquals(self.dbenvClient.rep_get_priority(), 0) self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL) # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE # is not generated if the master has no new transactions. # This is solved in BDB 4.6 (#15542). import time timeout = time.time() + 60 while (time.time() < timeout) and not (self.confirmed_master and self.client_startupdone): time.sleep(0.02) # self.client_startupdone does not always get set to True within # the timeout. On windows this may be a deep issue, on other # platforms it is likely just a timing issue, especially on slow # virthost buildbots (see issue 3892 for more). Even though # the timeout triggers, the rest of this test method usually passes # (but not all of it always, see below). So we just note the # timeout on stderr and keep soldering on. if time.time() > timeout: import sys print >> sys.stderr, ("XXX: timeout happened before" "startup was confirmed - see issue 3892") startup_timeout = True d = self.dbenvMaster.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], client_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) d = self.dbenvClient.repmgr_site_list() self.assertEquals(len(d), 1) self.assertEquals(d[0][0], "127.0.0.1") self.assertEquals(d[0][1], master_port) self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ (d[0][2]==db.DB_REPMGR_DISCONNECTED)) if db.version() >= (4, 6): d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR) self.assertTrue("msgs_queued" in d) self.dbMaster = db.DB(self.dbenvMaster) txn = self.dbenvMaster.txn_begin() self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) txn.commit() import time, os.path timeout = time.time() + 10 while (time.time()<timeout) and \ not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : time.sleep(0.01) self.dbClient = db.DB(self.dbenvClient) while True: txn = self.dbenvClient.txn_begin() try: self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn) except db.DBRepHandleDeadError: txn.abort() self.dbClient.close() self.dbClient = db.DB(self.dbenvClient) continue txn.commit() break txn = self.dbenvMaster.txn_begin() self.dbMaster.put("ABC", "123", txn=txn) txn.commit() import time timeout = time.time() + 10 v = None while (time.time() < timeout) and (v == None): txn = self.dbenvClient.txn_begin() v = self.dbClient.get("ABC", txn=txn) txn.commit() if v == None: time.sleep(0.02) # If startup did not happen before the timeout above, then this test # sometimes fails. This happens randomly, which causes buildbot # instability, but all the other bsddb tests pass. Since bsddb3 in the # stdlib is currently not getting active maintenance, and is gone in # py3k, we just skip the end of the test in that case. if time.time() >= timeout and startup_timeout: self.skipTest("replication test skipped due to random failure, " "see issue 3892") self.assertTrue(time.time() < timeout) self.assertEquals("123", v) txn = self.dbenvMaster.txn_begin() self.dbMaster.delete("ABC", txn=txn) txn.commit() timeout = time.time() + 10 while (time.time() < timeout) and (v != None): txn = self.dbenvClient.txn_begin() v = self.dbClient.get("ABC", txn=txn) txn.commit() if v == None: time.sleep(0.02) self.assertTrue(time.time() < timeout) self.assertEquals(None, v)