def test_suite():
suite = unittest.TestSuite()
if db.version() >= (4,5) :
suite.addTest(unittest.makeSuite(DBTxn_distributed))
suite.addTest(unittest.makeSuite(DBTxn_distributedSYNC))
if db.version() >= (4,6) :
suite.addTest(unittest.makeSuite(DBTxn_distributed_must_open_db))
suite.addTest(unittest.makeSuite(DBTxn_distributedSYNC_must_open_db))
return suite
def test07_DB_set_flags_persists(self):
if db.version() < (4,2):
# The get_flags API required for this to work is only available
# in Berkeley DB >= 4.2
return
try:
db1 = db.DB()
db1.set_flags(db.DB_DUPSORT)
db1.open(self.filename, db.DB_HASH, db.DB_CREATE)
db1['a'] = 'eh'
db1['a'] = 'A'
self.assertEqual([('a', 'A')], db1.items())
db1.put('a', 'Aa')
self.assertEqual([('a', 'A'), ('a', 'Aa')], db1.items())
db1.close()
db1 = db.DB()
# no set_flags call, we're testing that it reads and obeys
# the flags on open.
db1.open(self.filename, db.DB_HASH)
self.assertEqual([('a', 'A'), ('a', 'Aa')], db1.items())
# if it read the flags right this will replace all values
# for key 'a' instead of adding a new one. (as a dict should)
db1['a'] = 'new A'
self.assertEqual([('a', 'new A')], db1.items())
finally:
db1.close()
test_support.unlink(self.filename)
def test13_associate_in_transaction(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test13_associateAutoCommit..." % \
self.__class__.__name__
txn = self.env.txn_begin()
try:
self.createDB(txn=txn)
self.secDB = db.DB(self.env)
self.secDB.set_flags(db.DB_DUP)
self.secDB.set_get_returns_none(2)
self.secDB.open(self.filename, "secondary", db.DB_BTREE,
db.DB_CREATE | db.DB_THREAD, txn=txn)
if db.version() >= (4,1):
self.getDB().associate(self.secDB, self.getGenre, txn=txn)
else:
self.getDB().associate(self.secDB, self.getGenre)
self.addDataToDB(self.getDB(), txn=txn)
except:
txn.abort()
raise
self.txn_finish_test(self.secDB, txn=txn)
def test06_Transactions(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print "Running %s.test06_Transactions..." % self.__class__.__name__
self.assertEqual(d.get('new rec', txn=self.txn), None)
d.put('new rec', 'this is a new record', self.txn)
self.assertEqual(
d.get('new rec', txn=self.txn), 'this is a new record')
self.txn.abort()
self.assertEqual(d.get('new rec'), None)
self.txn = self.env.txn_begin()
self.assertEqual(d.get('new rec', txn=self.txn), None)
d.put('new rec', 'this is a new record', self.txn)
self.assertEqual(
d.get('new rec', txn=self.txn), 'this is a new record')
self.txn.commit()
self.assertEqual(d.get('new rec'), 'this is a new record')
self.txn = self.env.txn_begin()
c = d.cursor(self.txn)
rec = c.first()
count = 0
while rec is not None:
count = count + 1
if verbose and count % 100 == 0:
print rec
rec = c.next()
self.assertEqual(count, self._numKeys + 1)
c.close() # Cursors *MUST* be closed before commit!
self.txn.commit()
# flush pending updates
try:
self.env.txn_checkpoint(0, 0, 0)
except db.DBIncompleteError:
pass
statDict = self.env.log_stat(0)
self.assertTrue('magic' in statDict)
self.assertTrue('version' in statDict)
self.assertTrue('cur_file' in statDict)
self.assertTrue('region_nowait' in statDict)
# must have at least one log file present:
logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG)
self.assertNotEqual(logs, None)
for log in logs:
if verbose:
print 'log file: ' + log
if db.version() >= (4, 2):
logs = self.env.log_archive(db.DB_ARCH_REMOVE)
self.assertTrue(not logs)
self.txn = self.env.txn_begin()
def test00_version(self):
info = db.version()
if verbose:
print "\n", "-=" * 20
print "bsddb.db.version(): %s" % (info,)
print db.DB_VERSION_STRING
print "-=" * 20
self.assertEqual(info, (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, db.DB_VERSION_PATCH))
def test00_version(self):
info = db.version()
if verbose:
print '\n', '-=' * 20
print 'bsddb.db.version(): %s' % (info,)
print db.DB_VERSION_STRING
print '-=' * 20
self.assertEqual(info, (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, db.DB_VERSION_PATCH))
def createDB(self, txn=None):
self.cur = None
self.secDB = None
self.primary = db.DB(self.env)
self.primary.set_get_returns_none(2)
if db.version() >= (4, 1):
self.primary.open(self.filename, "primary", self.dbtype,
db.DB_CREATE | db.DB_THREAD | self.dbFlags, txn=txn)
else:
self.primary.open(self.filename, "primary", self.dbtype,
db.DB_CREATE | db.DB_THREAD | self.dbFlags)
def test_lk_detect(self) :
flags= [db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS,
db.DB_LOCK_MINLOCKS, db.DB_LOCK_MINWRITE,
db.DB_LOCK_OLDEST, db.DB_LOCK_RANDOM, db.DB_LOCK_YOUNGEST]
if db.version() >= (4, 3) :
flags.append(db.DB_LOCK_MAXWRITE)
for i in flags :
self.env.set_lk_detect(i)
self.assertEqual(i, self.env.get_lk_detect())
def test04_lock_timeout2(self):
self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT)
self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT)
self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT)
self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT)
def deadlock_detection() :
while not deadlock_detection.end :
deadlock_detection.count = \
self.env.lock_detect(db.DB_LOCK_EXPIRE)
if deadlock_detection.count :
while not deadlock_detection.end :
pass
break
time.sleep(0.01)
deadlock_detection.end=False
deadlock_detection.count=0
t=Thread(target=deadlock_detection)
import sys
if sys.version_info[0] < 3 :
t.setDaemon(True)
else :
t.daemon = True
t.start()
self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT)
anID = self.env.lock_id()
anID2 = self.env.lock_id()
self.assertNotEqual(anID, anID2)
lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE)
start_time=time.time()
self.assertRaises(db.DBLockNotGrantedError,
self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ)
end_time=time.time()
deadlock_detection.end=True
# Floating point rounding
if sys.platform == 'win32':
# bpo-30850: On Windows, tolerate 50 ms whereas 100 ms is expected.
# The lock sometimes times out after only 58 ms. Windows clocks
# have a bad resolution and bad accuracy.
min_dt = 0.050
else:
min_dt = 0.0999
self.assertGreaterEqual(end_time-start_time, min_dt)
self.env.lock_put(lock)
t.join()
self.env.lock_id_free(anID)
self.env.lock_id_free(anID2)
if db.version() >= (4,6):
self.assertGreater(deadlock_detection.count, 0)
def test_suite():
suite = unittest.TestSuite()
if db.version() >= (4, 6):
dbenv = db.DBEnv()
try:
dbenv.repmgr_get_ack_policy()
ReplicationManager_available = True
except:
ReplicationManager_available = False
dbenv.close()
del dbenv
if ReplicationManager_available:
suite.addTest(unittest.makeSuite(DBReplicationManager))
if have_threads:
suite.addTest(unittest.makeSuite(DBBaseReplication))
return suite
def setUp(self):
if self.useEnv:
self.homeDir=get_new_environment_path()
try:
self.env = db.DBEnv()
self.env.set_lg_max(1024*1024)
self.env.set_tx_max(30)
self._t = int(time.time())
self.env.set_tx_timestamp(self._t)
self.env.set_flags(self.envsetflags, 1)
self.env.open(self.homeDir, self.envflags | db.DB_CREATE)
self.filename = "test"
# Yes, a bare except is intended, since we're re-raising the exc.
except:
test_support.rmtree(self.homeDir)
raise
else:
self.env = None
self.filename = get_new_database_path()
# create and open the DB
self.d = db.DB(self.env)
if not self.useEnv :
if db.version() >= (4, 2) :
self.d.set_cachesize(*self.cachesize)
cachesize = self.d.get_cachesize()
self.assertEqual(cachesize[0], self.cachesize[0])
self.assertEqual(cachesize[2], self.cachesize[2])
# Berkeley DB expands the cache 25% accounting overhead,
# if the cache is small.
self.assertEqual(125, int(100.0*cachesize[1]/self.cachesize[1]))
self.d.set_flags(self.dbsetflags)
if self.dbname:
self.d.open(self.filename, self.dbname, self.dbtype,
self.dbopenflags|db.DB_CREATE, self.dbmode)
else:
self.d.open(self.filename, # try out keyword args
mode = self.dbmode,
dbtype = self.dbtype,
flags = self.dbopenflags|db.DB_CREATE)
if not self.useEnv:
self.assertRaises(db.DBInvalidArgError,
self.d.set_cachesize, *self.cachesize)
self.populateDB()
def test04_lock_timeout2(self):
self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT)
self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT)
self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT)
self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT)
def deadlock_detection() :
while not deadlock_detection.end :
deadlock_detection.count = \
self.env.lock_detect(db.DB_LOCK_EXPIRE)
if deadlock_detection.count :
while not deadlock_detection.end :
pass
break
time.sleep(0.01)
deadlock_detection.end=False
deadlock_detection.count=0
t=Thread(target=deadlock_detection)
import sys
if sys.version_info[0] < 3 :
t.setDaemon(True)
else :
t.daemon = True
t.start()
self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT)
anID = self.env.lock_id()
anID2 = self.env.lock_id()
self.assertNotEqual(anID, anID2)
lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE)
start_time=time.time()
self.assertRaises(db.DBLockNotGrantedError,
self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ)
end_time=time.time()
deadlock_detection.end=True
# Floating point rounding
self.assertTrue((end_time-start_time) >= 0.0999)
self.env.lock_put(lock)
t.join()
self.env.lock_id_free(anID)
self.env.lock_id_free(anID2)
if db.version() >= (4,6):
self.assertTrue(deadlock_detection.count>0)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(AssociateErrorTestCase))
suite.addTest(unittest.makeSuite(AssociateHashTestCase))
suite.addTest(unittest.makeSuite(AssociateBTreeTestCase))
suite.addTest(unittest.makeSuite(AssociateRecnoTestCase))
if db.version() >= (4, 1):
suite.addTest(unittest.makeSuite(AssociateBTreeTxnTestCase))
suite.addTest(unittest.makeSuite(ShelveAssociateHashTestCase))
suite.addTest(unittest.makeSuite(ShelveAssociateBTreeTestCase))
suite.addTest(unittest.makeSuite(ShelveAssociateRecnoTestCase))
if have_threads:
suite.addTest(unittest.makeSuite(ThreadedAssociateHashTestCase))
suite.addTest(unittest.makeSuite(ThreadedAssociateBTreeTestCase))
suite.addTest(unittest.makeSuite(ThreadedAssociateRecnoTestCase))
return suite
def test06_close_txn_before_dup_cursor(self) :
dbenv = db.DBEnv()
dbenv.open(self.homeDir,db.DB_INIT_TXN | db.DB_INIT_MPOOL |
db.DB_INIT_LOG | db.DB_CREATE)
d = db.DB(dbenv)
txn = dbenv.txn_begin()
if db.version() < (4,1) :
d.open(self.filename, dbtype = db.DB_HASH, flags = db.DB_CREATE)
else :
d.open(self.filename, dbtype = db.DB_HASH, flags = db.DB_CREATE,
txn=txn)
d.put("XXX", "yyy", txn=txn)
txn.commit()
txn = dbenv.txn_begin()
c1 = d.cursor(txn)
c2 = c1.dup()
self.assertEquals(("XXX", "yyy"), c1.first())
import warnings
# Not interested in warnings about implicit close.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
txn.commit()
self.assertRaises(db.DBCursorClosedError, c2.first)
def _create_env(self, must_open_db) :
self.dbenv = db.DBEnv()
self.dbenv.set_tx_max(self.num_txns)
self.dbenv.set_lk_max_lockers(self.num_txns*2)
self.dbenv.set_lk_max_locks(self.num_txns*2)
self.dbenv.set_lk_max_objects(self.num_txns*2)
if self.nosync :
self.dbenv.set_flags(db.DB_TXN_NOSYNC,True)
self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_THREAD |
db.DB_RECOVER |
db.DB_INIT_TXN | db.DB_INIT_LOG | db.DB_INIT_MPOOL |
db.DB_INIT_LOCK, 0666)
self.db = db.DB(self.dbenv)
self.db.set_re_len(db.DB_GID_SIZE)
if must_open_db :
if db.version() >= (4,2) :
txn=self.dbenv.txn_begin()
self.db.open(self.filename,
db.DB_QUEUE, db.DB_CREATE | db.DB_THREAD, 0666,
txn=txn)
txn.commit()
else :
self.db.open(self.filename,
db.DB_QUEUE, db.DB_CREATE | db.DB_THREAD, 0666)
def test02_basicPost32(self):
# Basic Queue tests using the new DB.consume method in DB 3.2+
# (No cursor needed)
if verbose:
print '\n', '-=' * 30
print "Running %s.test02_basicPost32..." % self.__class__.__name__
if db.version() < (3, 2, 0):
if verbose:
print "Test not run, DB not new enough..."
return
d = db.DB()
d.set_re_len(40) # Queues must be fixed length
d.open(self.filename, db.DB_QUEUE, db.DB_CREATE)
if verbose:
print "before appends" + '-' * 30
pprint(d.stat())
for x in string.letters:
d.append(x * 40)
self.assertEqual(len(d), len(string.letters))
d.put(100, "some more data")
d.put(101, "and some more ")
d.put(75, "out of order")
d.put(1, "replacement data")
self.assertEqual(len(d), len(string.letters) + 3)
if verbose:
print "before close" + '-' * 30
pprint(d.stat())
d.close()
del d
d = db.DB()
d.open(self.filename)
#d.set_get_returns_none(true)
if verbose:
print "after open" + '-' * 30
pprint(d.stat())
d.append("one more")
if verbose:
print "after append" + '-' * 30
pprint(d.stat())
rec = d.consume()
while rec:
if verbose:
print rec
rec = d.consume()
if verbose:
print "after consume loop" + '-' * 30
pprint(d.stat())
d.close()
def test02_basicPost32(self):
# Basic Queue tests using the new DB.consume method in DB 3.2+
# (No cursor needed)
if verbose:
print '\n', '-=' * 30
print "Running %s.test02_basicPost32..." % self.__class__.__name__
if db.version() < (3, 2, 0):
if verbose:
print "Test not run, DB not new enough..."
return
d = db.DB()
d.set_re_len(40) # Queues must be fixed length
d.open(self.filename, db.DB_QUEUE, db.DB_CREATE)
if verbose:
print "before appends" + '-' * 30
pprint(d.stat())
for x in string.letters:
d.append(x * 40)
self.assertEqual(len(d), len(string.letters))
d.put(100, "some more data")
d.put(101, "and some more ")
d.put(75, "out of order")
d.put(1, "replacement data")
self.assertEqual(len(d), len(string.letters)+3)
if verbose:
print "before close" + '-' * 30
pprint(d.stat())
d.close()
del d
d = db.DB()
d.open(self.filename)
#d.set_get_returns_none(true)
if verbose:
print "after open" + '-' * 30
pprint(d.stat())
d.append("one more")
if verbose:
print "after append" + '-' * 30
pprint(d.stat())
rec = d.consume()
while rec:
if verbose:
print rec
rec = d.consume()
if verbose:
print "after consume loop" + '-' * 30
pprint(d.stat())
d.close()
class DBBaseReplication(DBReplication):
def setUp(self):
DBReplication.setUp(self)
def confirmed_master(a, b, c):
if b == db.DB_EVENT_REP_MASTER or b == db.DB_EVENT_REP_ELECTED:
self.confirmed_master = True
def client_startupdone(a, b, c):
if b == db.DB_EVENT_REP_STARTUPDONE:
self.client_startupdone = True
self.dbenvMaster.set_event_notify(confirmed_master)
self.dbenvClient.set_event_notify(client_startupdone)
import Queue
self.m2c = Queue.Queue()
self.c2m = Queue.Queue()
def m2c(dbenv, control, rec, lsnp, envid, flags):
self.m2c.put((control, rec))
def c2m(dbenv, control, rec, lsnp, envid, flags):
self.c2m.put((control, rec))
self.dbenvMaster.rep_set_transport(13, m2c)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_transport(3, c2m)
self.dbenvClient.rep_set_priority(0)
self.assertEqual(self.dbenvMaster.rep_get_priority(), 10)
self.assertEqual(self.dbenvClient.rep_get_priority(), 0)
def thread_master():
return self.thread_do(self.dbenvMaster, self.c2m, 3,
self.master_doing_election, True)
def thread_client():
return self.thread_do(self.dbenvClient, self.m2c, 13,
self.client_doing_election, False)
from threading import Thread
t_m = Thread(target=thread_master)
t_c = Thread(target=thread_client)
import sys
if sys.version_info[0] < 3:
t_m.setDaemon(True)
t_c.setDaemon(True)
else:
t_m.daemon = True
t_c.daemon = True
self.t_m = t_m
self.t_c = t_c
self.dbMaster = self.dbClient = None
self.master_doing_election = [False]
self.client_doing_election = [False]
return
def tearDown(self):
if self.dbClient:
self.dbClient.close()
if self.dbMaster:
self.dbMaster.close()
self.m2c.put(None)
self.c2m.put(None)
self.t_m.join()
self.t_c.join()
def dummy(*args):
pass
self.dbenvMaster.set_event_notify(dummy)
self.dbenvClient.set_event_notify(dummy)
self.dbenvMaster.rep_set_transport(13, dummy)
self.dbenvClient.rep_set_transport(3, dummy)
self.dbenvClient.close()
self.dbenvMaster.close()
test_support.rmtree(self.homeDirClient)
test_support.rmtree(self.homeDirMaster)
return
def basic_rep_threading(self):
self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER)
self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT)
def thread_do(env, q, envid, election_status, must_be_master):
while True:
v = q.get()
if v is None:
return
env.rep_process_message(v[0], v[1], envid)
return
self.thread_do = thread_do
self.t_m.start()
self.t_c.start()
def test01_basic_replication(self):
self.basic_rep_threading()
import time
timeout = time.time() + 60
while time.time() < timeout and not (self.confirmed_master
and self.client_startupdone):
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.dbMaster = db.DB(self.dbenvMaster)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.open('test', db.DB_HASH, db.DB_CREATE, 438, txn=txn)
txn.commit()
import time, os.path
timeout = time.time() + 10
while time.time() < timeout and not os.path.exists(
os.path.join(self.homeDirClient, 'test')):
time.sleep(0.01)
self.dbClient = db.DB(self.dbenvClient)
while True:
txn = self.dbenvClient.txn_begin()
try:
self.dbClient.open('test',
db.DB_HASH,
flags=db.DB_RDONLY,
mode=438,
txn=txn)
except db.DBRepHandleDeadError:
txn.abort()
self.dbClient.close()
self.dbClient = db.DB(self.dbenvClient)
continue
txn.commit()
break
d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR)
self.assertTrue('master_changes' in d)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.put('ABC', '123', txn=txn)
txn.commit()
import time
timeout = time.time() + 10
v = None
while time.time() < timeout and v is None:
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get('ABC', txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEqual('123', v)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.delete('ABC', txn=txn)
txn.commit()
timeout = time.time() + 10
while time.time() < timeout and v is not None:
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get('ABC', txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEqual(None, v)
return
if db.version() >= (4, 7):
def test02_test_request(self):
self.basic_rep_threading()
minimum, maximum = self.dbenvClient.rep_get_request()
self.dbenvClient.rep_set_request(minimum - 1, maximum + 1)
self.assertEqual(self.dbenvClient.rep_get_request(),
(minimum - 1, maximum + 1))
if db.version() >= (4, 6):
def test03_master_election(self):
self.dbenvMaster.rep_start(flags=db.DB_REP_CLIENT)
self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT)
def thread_do(env, q, envid, election_status, must_be_master):
while True:
v = q.get()
if v is None:
return
r = env.rep_process_message(v[0], v[1], envid)
if must_be_master and self.confirmed_master:
self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER)
must_be_master = False
if r[0] == db.DB_REP_HOLDELECTION:
def elect():
while True:
try:
env.rep_elect(2, 1)
election_status[0] = False
break
except db.DBRepUnavailError:
pass
if not election_status[0] and not self.confirmed_master:
from threading import Thread
election_status[0] = True
t = Thread(target=elect)
import sys
if sys.version_info[0] < 3:
t.setDaemon(True)
else:
t.daemon = True
t.start()
return
self.thread_do = thread_do
self.t_m.start()
self.t_c.start()
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000)
self.client_doing_election[0] = True
while True:
try:
self.dbenvClient.rep_elect(2, 1)
self.client_doing_election[0] = False
break
except db.DBRepUnavailError:
pass
self.assertTrue(self.confirmed_master)
while not self.dbenvClient.rep_stat()['startup_complete']:
pass
if db.version() >= (4, 7):
def test04_test_clockskew(self):
fast, slow = (1234, 1230)
self.dbenvMaster.rep_set_clockskew(fast, slow)
self.assertEqual((fast, slow),
self.dbenvMaster.rep_get_clockskew())
self.basic_rep_threading()
def test01_basic_replication(self):
master_port = test_support.find_unused_port()
client_port = test_support.find_unused_port()
if db.version() >= (5, 2):
self.site = self.dbenvMaster.repmgr_site("127.0.0.1", master_port)
self.site.set_config(db.DB_GROUP_CREATOR, True)
self.site.set_config(db.DB_LOCAL_SITE, True)
self.site2 = self.dbenvMaster.repmgr_site("127.0.0.1", client_port)
self.site3 = self.dbenvClient.repmgr_site("127.0.0.1", master_port)
self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True)
self.site4 = self.dbenvClient.repmgr_site("127.0.0.1", client_port)
self.site4.set_config(db.DB_LOCAL_SITE, True)
d = {
db.DB_BOOTSTRAP_HELPER: [False, False, True, False],
db.DB_GROUP_CREATOR: [True, False, False, False],
db.DB_LEGACY: [False, False, False, False],
db.DB_LOCAL_SITE: [True, False, False, True],
db.DB_REPMGR_PEER: [False, False, False, False],
}
for i, j in d.items():
for k, v in zip([self.site, self.site2, self.site3, self.site4], j):
if v:
self.assertTrue(k.get_config(i))
else:
self.assertFalse(k.get_config(i))
self.assertNotEqual(self.site.get_eid(), self.site2.get_eid())
self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid())
for i, j in zip(
[self.site, self.site2, self.site3, self.site4], [master_port, client_port, master_port, client_port]
):
addr = i.get_address()
self.assertEqual(addr, ("127.0.0.1", j))
for i in [self.site, self.site2]:
self.assertEqual(i.get_address(), self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address())
for i in [self.site3, self.site4]:
self.assertEqual(i.get_address(), self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address())
else:
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321)
self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123)
self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234)
self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345)
self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER)
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT)
self.assertEqual(self.dbenvMaster.rep_get_nsites(), 2)
self.assertEqual(self.dbenvClient.rep_get_nsites(), 2)
self.assertEqual(self.dbenvMaster.rep_get_priority(), 10)
self.assertEqual(self.dbenvClient.rep_get_priority(), 0)
self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL)
self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time() + 10
while (time.time() < timeout) and not (self.confirmed_master and self.client_startupdone):
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
d = self.dbenvMaster.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0] # There is only one
self.assertEqual(d[0], "127.0.0.1")
self.assertEqual(d[1], client_port)
self.assertTrue((d[2] == db.DB_REPMGR_CONNECTED) or (d[2] == db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0] # There is only one
self.assertEqual(d[0], "127.0.0.1")
self.assertEqual(d[1], master_port)
self.assertTrue((d[2] == db.DB_REPMGR_CONNECTED) or (d[2] == db.DB_REPMGR_DISCONNECTED))
if db.version() >= (4, 6):
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR)
self.assertTrue("msgs_queued" in d)
self.dbMaster = db.DB(self.dbenvMaster)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time, os.path
timeout = time.time() + 10
while (time.time() < timeout) and not (os.path.exists(os.path.join(self.homeDirClient, "test"))):
time.sleep(0.01)
self.dbClient = db.DB(self.dbenvClient)
while True:
txn = self.dbenvClient.txn_begin()
try:
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, mode=0666, txn=txn)
except db.DBRepHandleDeadError:
txn.abort()
self.dbClient.close()
self.dbClient = db.DB(self.dbenvClient)
continue
txn.commit()
break
txn = self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout = time.time() + 10
v = None
while (time.time() < timeout) and (v is None):
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get("ABC", txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEqual("123", v)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout = time.time() + 10
while (time.time() < timeout) and (v is not None):
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get("ABC", txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEqual(None, v)
def test01_basic_replication(self):
master_port = test_support.find_unused_port()
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
client_port = test_support.find_unused_port()
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321)
self.assertEquals(
self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY),
100123)
self.assertEquals(
self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY),
100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEquals(
self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT),
100234)
self.assertEquals(
self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT),
100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEquals(
self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345)
self.assertEquals(
self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER)
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT)
self.assertEquals(self.dbenvMaster.rep_get_nsites(), 2)
self.assertEquals(self.dbenvClient.rep_get_nsites(), 2)
self.assertEquals(self.dbenvMaster.rep_get_priority(), 10)
self.assertEquals(self.dbenvClient.rep_get_priority(), 0)
self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time() + 60
while (time.time() < timeout) and not (self.confirmed_master
and self.client_startupdone):
time.sleep(0.02)
# this fails on Windows as self.client_startupdone never gets set
# to True - see bug 3892. BUT - even though this assertion
# fails on Windows the rest of the test passes - so to prove
# that we let the rest of the test run. Sadly we can't
# make use of raising TestSkipped() here (unittest still
# reports it as an error), so we yell to stderr.
import sys
if sys.platform == "win32":
print >> sys.stderr, \
"XXX - windows bsddb replication fails on windows and is skipped"
print >> sys.stderr, "XXX - Please see issue #3892"
# It also fails irregularly on other platforms, and again the
# rest of the tests pass. Since bsddb support is unmaintained, and
# is gone in py3k, we just emit a warning instead of a test failure
# so as to improve buildbot stability.
elif time.time() > timeout:
print >> sys.stderr, \
"XXX - timeout before startup confirmed, see issue #3892."
d = self.dbenvMaster.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], client_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], master_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
if db.version() >= (4, 6):
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR)
self.assertTrue("msgs_queued" in d)
self.dbMaster = db.DB(self.dbenvMaster)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time, os.path
timeout = time.time() + 10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient = db.DB(self.dbenvClient)
while True:
txn = self.dbenvClient.txn_begin()
try:
self.dbClient.open("test",
db.DB_HASH,
flags=db.DB_RDONLY,
mode=0666,
txn=txn)
except db.DBRepHandleDeadError:
txn.abort()
self.dbClient.close()
self.dbClient = db.DB(self.dbenvClient)
continue
txn.commit()
break
txn = self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout = time.time() + 10
v = None
while (time.time() < timeout) and (v == None):
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get("ABC", txn=txn)
txn.commit()
if v == None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEquals("123", v)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout = time.time() + 10
while (time.time() < timeout) and (v != None):
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get("ABC", txn=txn)
txn.commit()
if v == None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEquals(None, v)
self.assertTrue(size <= size2[1])
self.assertTrue(2*size > size2[1])
if db.version() >= (4, 4) :
def test_mutex_stat(self) :
self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL |
db.DB_INIT_LOCK)
stat = self.env.mutex_stat()
self.assertTrue("mutex_inuse_max" in stat)
def test_lg_filemode(self) :
for i in [0600, 0660, 0666] :
self.env.set_lg_filemode(i)
self.assertEqual(i, self.env.get_lg_filemode())
if db.version() >= (4, 3) :
def test_mp_max_openfd(self) :
for i in [17, 31, 42] :
self.env.set_mp_max_openfd(i)
self.assertEqual(i, self.env.get_mp_max_openfd())
def test_mp_max_write(self) :
for i in [100, 200, 300] :
for j in [1, 2, 3] :
j *= 1000000
self.env.set_mp_max_write(i, j)
v=self.env.get_mp_max_write()
self.assertEqual((i, j), v)
if db.version() >= (4, 2) :
def test_invalid_txn(self) :
def test01_basic_replication(self) :
self.basic_rep_threading()
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time()+10
while (time.time()<timeout) and not (self.confirmed_master and
self.client_startupdone) :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.dbMaster=db.DB(self.dbenvMaster)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time,os.path
timeout=time.time()+10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient=db.DB(self.dbenvClient)
while True :
txn=self.dbenvClient.txn_begin()
try :
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY,
mode=0666, txn=txn)
except db.DBRepHandleDeadError :
txn.abort()
self.dbClient.close()
self.dbClient=db.DB(self.dbenvClient)
continue
txn.commit()
break
if db.version() >= (4,1) :
d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR);
self.assertTrue("master_changes" in d)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout=time.time()+10
v=None
while (time.time()<timeout) and (v==None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals("123", v)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout=time.time()+10
while (time.time()<timeout) and (v!=None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals(None, v)
self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=1)
def test03c_SimpleCursorGetReturnsNone2(self):
# same test but raise exceptions instead of returning None
if verbose:
print '\n', '-=' * 30
print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \
self.__class__.__name__
old = self.d.set_get_returns_none(1)
self.assertEqual(old, 2)
old = self.d.set_get_returns_none(2)
self.assertEqual(old, 1)
self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0)
if db.version() >= (4, 6):
def test03d_SimpleCursorPriority(self):
c = self.d.cursor()
c.set_priority(db.DB_PRIORITY_VERY_LOW) # Positional
self.assertEqual(db.DB_PRIORITY_VERY_LOW, c.get_priority())
c.set_priority(priority=db.DB_PRIORITY_HIGH) # Keyword
self.assertEqual(db.DB_PRIORITY_HIGH, c.get_priority())
c.close()
#----------------------------------------
def test04_PartialGetAndPut(self):
d = self.d
if verbose:
print '\n', '-=' * 30
shm_key=137
self.env.set_shm_key(shm_key)
self.assertEqual(shm_key, self.env.get_shm_key())
self.env.set_shm_key(shm_key+1)
self.assertEqual(shm_key+1, self.env.get_shm_key())
# You can not change configuration after opening
# the environment.
self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL)
# If we try to reconfigure cache after opening the
# environment, core dump.
self.assertRaises(db.DBInvalidArgError,
self.env.set_shm_key, shm_key)
self.assertEqual(shm_key+1, self.env.get_shm_key())
if db.version() >= (4, 4) :
def test_mutex_setget_max(self) :
v = self.env.mutex_get_max()
v2 = v*2+1
self.env.mutex_set_max(v2)
self.assertEqual(v2, self.env.mutex_get_max())
self.env.mutex_set_max(v)
self.assertEqual(v, self.env.mutex_get_max())
# You can not change configuration after opening
# the environment.
self.env.open(self.homeDir, db.DB_CREATE)
self.assertRaises(db.DBInvalidArgError,
self.env.mutex_set_max, v2)
def _destroy_env(self):
if self.nosync or (db.version()[:2] == (4, 6)): # Known bug
self.dbenv.log_flush()
self.db.close()
self.dbenv.close()
class DBEnvClosedEarlyCrash(unittest.TestCase):
def setUp(self):
self.homeDir = get_new_environment_path()
self.filename = "test"
def tearDown(self):
test_support.rmtree(self.homeDir)
def test01_close_dbenv_before_db(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
d = db.DB(dbenv)
d2 = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
self.assertRaises(db.DBNoSuchFileError, d2.open, self.filename + "2",
db.DB_BTREE, db.DB_THREAD, 0666)
d.put("test", "this is a test")
self.assertEqual(d.get("test"), "this is a test", "put!=get")
dbenv.close() # This "close" should close the child db handle also
self.assertRaises(db.DBError, d.get, "test")
def test02_close_dbenv_before_dbcursor(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
d = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
d.put("test", "this is a test")
d.put("test2", "another test")
d.put("test3", "another one")
self.assertEqual(d.get("test"), "this is a test", "put!=get")
c = d.cursor()
c.first()
c.next()
d.close() # This "close" should close the child db handle also
# db.close should close the child cursor
self.assertRaises(db.DBError, c.next)
d = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
c = d.cursor()
c.first()
c.next()
dbenv.close()
# The "close" should close the child db handle also, with cursors
self.assertRaises(db.DBError, c.next)
def test03_close_db_before_dbcursor_without_env(self):
import os.path
path = os.path.join(self.homeDir, self.filename)
d = db.DB()
d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
d.put("test", "this is a test")
d.put("test2", "another test")
d.put("test3", "another one")
self.assertEqual(d.get("test"), "this is a test", "put!=get")
c = d.cursor()
c.first()
c.next()
d.close()
# The "close" should close the child db handle also
self.assertRaises(db.DBError, c.next)
def test04_close_massive(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
dbs = [db.DB(dbenv) for i in xrange(16)]
cursors = []
for i in dbs:
i.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD,
0666)
dbs[10].put("test", "this is a test")
dbs[10].put("test2", "another test")
dbs[10].put("test3", "another one")
self.assertEqual(dbs[4].get("test"), "this is a test", "put!=get")
for i in dbs:
cursors.extend([i.cursor() for j in xrange(32)])
for i in dbs[::3]:
i.close()
for i in cursors[::3]:
i.close()
# Check for missing exception in DB! (after DB close)
self.assertRaises(db.DBError, dbs[9].get, "test")
# Check for missing exception in DBCursor! (after DB close)
self.assertRaises(db.DBError, cursors[101].first)
cursors[80].first()
cursors[80].next()
dbenv.close() # This "close" should close the child db handle also
# Check for missing exception! (after DBEnv close)
self.assertRaises(db.DBError, cursors[80].next)
def test05_close_dbenv_delete_db_success(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
d = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
dbenv.close() # This "close" should close the child db handle also
del d
try:
import gc
except ImportError:
gc = None
if gc:
# force d.__del__ [DB_dealloc] to be called
gc.collect()
def test06_close_txn_before_dup_cursor(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_TXN | db.DB_INIT_MPOOL | db.DB_INIT_LOG | db.DB_CREATE)
d = db.DB(dbenv)
txn = dbenv.txn_begin()
d.open(self.filename, dbtype=db.DB_HASH, flags=db.DB_CREATE, txn=txn)
d.put("XXX", "yyy", txn=txn)
txn.commit()
txn = dbenv.txn_begin()
c1 = d.cursor(txn)
c2 = c1.dup()
self.assertEquals(("XXX", "yyy"), c1.first())
# Not interested in warnings about implicit close.
import warnings
if sys.version_info < (2, 6):
# Completely resetting the warning state is
# problematic with python >=2.6 with -3 (py3k warning),
# because some stdlib modules selectively ignores warnings.
warnings.simplefilter("ignore")
txn.commit()
warnings.resetwarnings()
else:
# When we drop support for python 2.3 and 2.4
# we could use: (in 2.5 we need a __future__ statement)
#
# with warnings.catch_warnings():
# warnings.simplefilter("ignore")
# txn.commit()
#
# We can not use "with" as is, because it would be invalid syntax
# in python 2.3, 2.4 and (with no __future__) 2.5.
# Here we simulate "with" following PEP 343 :
w = warnings.catch_warnings()
w.__enter__()
try:
warnings.simplefilter("ignore")
txn.commit()
finally:
w.__exit__()
self.assertRaises(db.DBCursorClosedError, c2.first)
if db.version() > (4, 3, 0):
def test07_close_db_before_sequence(self):
import os.path
path = os.path.join(self.homeDir, self.filename)
d = db.DB()
d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
dbs = db.DBSequence(d)
d.close() # This "close" should close the child DBSequence also
dbs.close() # If not closed, core dump (in Berkeley DB 4.6.*)
shm_key = 137
self.env.set_shm_key(shm_key)
self.assertEqual(shm_key, self.env.get_shm_key())
self.env.set_shm_key(shm_key + 1)
self.assertEqual(shm_key + 1, self.env.get_shm_key())
# You can not change configuration after opening
# the environment.
self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL)
# If we try to reconfigure cache after opening the
# environment, core dump.
self.assertRaises(db.DBInvalidArgError, self.env.set_shm_key,
shm_key)
self.assertEqual(shm_key + 1, self.env.get_shm_key())
if db.version() >= (4, 4):
def test_mutex_setget_max(self):
v = self.env.mutex_get_max()
v2 = v * 2 + 1
self.env.mutex_set_max(v2)
self.assertEqual(v2, self.env.mutex_get_max())
self.env.mutex_set_max(v)
self.assertEqual(v, self.env.mutex_get_max())
# You can not change configuration after opening
# the environment.
self.env.open(self.homeDir, db.DB_CREATE)
self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_max, v2)
self.assertTrue(size <= size2[1])
self.assertTrue(2*size > size2[1])
if db.version() >= (4, 4) :
def test_mutex_stat(self) :
self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL |
db.DB_INIT_LOCK)
stat = self.env.mutex_stat()
self.assertTrue("mutex_inuse_max" in stat)
def test_lg_filemode(self) :
for i in [0600, 0660, 0666] :
self.env.set_lg_filemode(i)
self.assertEqual(i, self.env.get_lg_filemode())
if db.version() >= (4, 3) :
def test_mp_max_openfd(self) :
for i in [17, 31, 42] :
self.env.set_mp_max_openfd(i)
self.assertEqual(i, self.env.get_mp_max_openfd())
def test_mp_max_write(self) :
for i in [100, 200, 300] :
for j in [1, 2, 3] :
j *= 1000000
self.env.set_mp_max_write(i, j)
v=self.env.get_mp_max_write()
self.assertEqual((i, j), v)
if db.version() >= (4, 2) :
def test_invalid_txn(self) :
class AssociateTestCase(unittest.TestCase):
keytype = ''
envFlags = 0
dbFlags = 0
def setUp(self):
self.filename = self.__class__.__name__ + '.db'
self.homeDir = get_new_environment_path()
self.env = db.DBEnv()
self.env.open(
self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK
| db.DB_THREAD | self.envFlags)
def tearDown(self):
self.closeDB()
self.env.close()
self.env = None
test_support.rmtree(self.homeDir)
def addDataToDB(self, d, txn=None):
for key, value in musicdata.items():
if type(self.keytype) == type(''):
key = "%02d" % key
d.put(key, '|'.join(value), txn=txn)
def createDB(self, txn=None):
self.cur = None
self.secDB = None
self.primary = db.DB(self.env)
self.primary.set_get_returns_none(2)
self.primary.open(self.filename,
"primary",
self.dbtype,
db.DB_CREATE | db.DB_THREAD | self.dbFlags,
txn=txn)
def closeDB(self):
if self.cur:
self.cur.close()
self.cur = None
if self.secDB:
self.secDB.close()
self.secDB = None
self.primary.close()
self.primary = None
def getDB(self):
return self.primary
def _associateWithDB(self, getGenre):
self.createDB()
self.secDB = db.DB(self.env)
self.secDB.set_flags(db.DB_DUP)
self.secDB.set_get_returns_none(2)
self.secDB.open(self.filename, "secondary", db.DB_BTREE,
db.DB_CREATE | db.DB_THREAD | self.dbFlags)
self.getDB().associate(self.secDB, getGenre)
self.addDataToDB(self.getDB())
self.finish_test(self.secDB)
def test01_associateWithDB(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test01_associateWithDB..." % \
self.__class__.__name__
return self._associateWithDB(self.getGenre)
def _associateAfterDB(self, getGenre):
self.createDB()
self.addDataToDB(self.getDB())
self.secDB = db.DB(self.env)
self.secDB.set_flags(db.DB_DUP)
self.secDB.open(self.filename, "secondary", db.DB_BTREE,
db.DB_CREATE | db.DB_THREAD | self.dbFlags)
# adding the DB_CREATE flag will cause it to index existing records
self.getDB().associate(self.secDB, getGenre, db.DB_CREATE)
self.finish_test(self.secDB)
def test02_associateAfterDB(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test02_associateAfterDB..." % \
self.__class__.__name__
return self._associateAfterDB(self.getGenre)
if db.version() >= (4, 6):
def test03_associateWithDB(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test03_associateWithDB..." % \
self.__class__.__name__
return self._associateWithDB(self.getGenreList)
def test04_associateAfterDB(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test04_associateAfterDB..." % \
self.__class__.__name__
return self._associateAfterDB(self.getGenreList)
def finish_test(self, secDB, txn=None):
# 'Blues' should not be in the secondary database
vals = secDB.pget('Blues', txn=txn)
self.assertEqual(vals, None, vals)
vals = secDB.pget('Unknown', txn=txn)
self.assert_(vals[0] == 99 or vals[0] == '99', vals)
vals[1].index('Unknown')
vals[1].index('Unnamed')
vals[1].index('unknown')
if verbose:
print "Primary key traversal:"
self.cur = self.getDB().cursor(txn)
count = 0
rec = self.cur.first()
while rec is not None:
if type(self.keytype) == type(''):
self.assert_(int(rec[0])) # for primary db, key is a number
else:
self.assert_(rec[0] and type(rec[0]) == type(0))
count = count + 1
if verbose:
print rec
rec = getattr(self.cur, "next")()
self.assertEqual(count, len(musicdata)) # all items accounted for
if verbose:
print "Secondary key traversal:"
self.cur = secDB.cursor(txn)
count = 0
# test cursor pget
vals = self.cur.pget('Unknown', flags=db.DB_LAST)
self.assert_(vals[1] == 99 or vals[1] == '99', vals)
self.assertEqual(vals[0], 'Unknown')
vals[2].index('Unknown')
vals[2].index('Unnamed')
vals[2].index('unknown')
vals = self.cur.pget('Unknown',
data='wrong value',
flags=db.DB_GET_BOTH)
self.assertEqual(vals, None, vals)
rec = self.cur.first()
self.assertEqual(rec[0], "Jazz")
while rec is not None:
count = count + 1
if verbose:
print rec
rec = getattr(self.cur, "next")()
# all items accounted for EXCEPT for 1 with "Blues" genre
self.assertEqual(count, len(musicdata) - 1)
self.cur = None
def getGenre(self, priKey, priData):
self.assertEqual(type(priData), type(""))
genre = priData.split('|')[2]
if verbose:
print 'getGenre key: %r data: %r' % (priKey, priData)
if genre == 'Blues':
return db.DB_DONOTINDEX
else:
return genre
def getGenreList(self, priKey, PriData):
v = self.getGenre(priKey, PriData)
if type(v) == type(""):
v = [v]
return v
def test_suite():
suite = unittest.TestSuite()
if db.version() >= (4, 4):
suite.addTest(unittest.makeSuite(FileidResetTestCase))
return suite
def test01_basic_replication(self) :
master_port = test_support.find_unused_port()
client_port = test_support.find_unused_port()
if db.version() >= (5, 2) :
self.site = self.dbenvMaster.repmgr_site("127.0.0.1", master_port)
self.site.set_config(db.DB_GROUP_CREATOR, True)
self.site.set_config(db.DB_LOCAL_SITE, True)
self.site2 = self.dbenvMaster.repmgr_site("127.0.0.1", client_port)
self.site3 = self.dbenvClient.repmgr_site("127.0.0.1", master_port)
self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True)
self.site4 = self.dbenvClient.repmgr_site("127.0.0.1", client_port)
self.site4.set_config(db.DB_LOCAL_SITE, True)
d = {
db.DB_BOOTSTRAP_HELPER: [False, False, True, False],
db.DB_GROUP_CREATOR: [True, False, False, False],
db.DB_LEGACY: [False, False, False, False],
db.DB_LOCAL_SITE: [True, False, False, True],
db.DB_REPMGR_PEER: [False, False, False, False ],
}
for i, j in d.items() :
for k, v in \
zip([self.site, self.site2, self.site3, self.site4], j) :
if v :
self.assertTrue(k.get_config(i))
else :
self.assertFalse(k.get_config(i))
self.assertNotEqual(self.site.get_eid(), self.site2.get_eid())
self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid())
for i, j in zip([self.site, self.site2, self.site3, self.site4], \
[master_port, client_port, master_port, client_port]) :
addr = i.get_address()
self.assertEqual(addr, ("127.0.0.1", j))
for i in [self.site, self.site2] :
self.assertEqual(i.get_address(),
self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address())
for i in [self.site3, self.site4] :
self.assertEqual(i.get_address(),
self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address())
else :
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321)
self.assertEqual(self.dbenvMaster.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100123)
self.assertEqual(self.dbenvClient.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEqual(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100234)
self.assertEqual(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEqual(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100345)
self.assertEqual(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER);
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT);
self.assertEqual(self.dbenvMaster.rep_get_nsites(),2)
self.assertEqual(self.dbenvClient.rep_get_nsites(),2)
self.assertEqual(self.dbenvMaster.rep_get_priority(),10)
self.assertEqual(self.dbenvClient.rep_get_priority(),0)
self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time()+60
while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) :
time.sleep(0.02)
# self.client_startupdone does not always get set to True within
# the timeout. On windows this may be a deep issue, on other
# platforms it is likely just a timing issue, especially on slow
# virthost buildbots (see issue 3892 for more). Even though
# the timeout triggers, the rest of this test method usually passes
# (but not all of it always, see below). So we just note the
# timeout on stderr and keep soldering on.
if time.time()>timeout:
import sys
print >> sys.stderr, ("XXX: timeout happened before"
"startup was confirmed - see issue 3892")
startup_timeout = True
d = self.dbenvMaster.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0] # There is only one
self.assertEqual(d[0], "127.0.0.1")
self.assertEqual(d[1], client_port)
self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
(d[2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0] # There is only one
self.assertEqual(d[0], "127.0.0.1")
self.assertEqual(d[1], master_port)
self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
(d[2]==db.DB_REPMGR_DISCONNECTED))
if db.version() >= (4,6) :
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
self.assertTrue("msgs_queued" in d)
self.dbMaster=db.DB(self.dbenvMaster)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time,os.path
timeout=time.time()+10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient=db.DB(self.dbenvClient)
while True :
txn=self.dbenvClient.txn_begin()
try :
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY,
mode=0666, txn=txn)
except db.DBRepHandleDeadError :
txn.abort()
self.dbClient.close()
self.dbClient=db.DB(self.dbenvClient)
continue
txn.commit()
break
txn=self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout=time.time()+10
v=None
while (time.time()<timeout) and (v is None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v is None :
time.sleep(0.02)
# If startup did not happen before the timeout above, then this test
# sometimes fails. This happens randomly, which causes buildbot
# instability, but all the other bsddb tests pass. Since bsddb3 in the
# stdlib is currently not getting active maintenance, and is gone in
# py3k, we just skip the end of the test in that case.
if time.time()>=timeout and startup_timeout:
self.skipTest("replication test skipped due to random failure, "
"see issue 3892")
self.assertTrue(time.time()<timeout)
self.assertEqual("123", v)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout=time.time()+10
while (time.time()<timeout) and (v is not None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v is None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEqual(None, v)
def test01_basic_replication(self):
master_port = test_support.find_unused_port()
client_port = test_support.find_unused_port()
if db.version() >= (5, 2):
self.site = self.dbenvMaster.repmgr_site('127.0.0.1', master_port)
self.site.set_config(db.DB_GROUP_CREATOR, True)
self.site.set_config(db.DB_LOCAL_SITE, True)
self.site2 = self.dbenvMaster.repmgr_site('127.0.0.1', client_port)
self.site3 = self.dbenvClient.repmgr_site('127.0.0.1', master_port)
self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True)
self.site4 = self.dbenvClient.repmgr_site('127.0.0.1', client_port)
self.site4.set_config(db.DB_LOCAL_SITE, True)
d = {
db.DB_BOOTSTRAP_HELPER: [False, False, True, False],
db.DB_GROUP_CREATOR: [True, False, False, False],
db.DB_LEGACY: [False, False, False, False],
db.DB_LOCAL_SITE: [True, False, False, True],
db.DB_REPMGR_PEER: [False, False, False, False]
}
for i, j in d.items():
for k, v in zip(
[self.site, self.site2, self.site3, self.site4], j):
if v:
self.assertTrue(k.get_config(i))
else:
self.assertFalse(k.get_config(i))
self.assertNotEqual(self.site.get_eid(), self.site2.get_eid())
self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid())
for i, j in zip(
[self.site, self.site2, self.site3, self.site4],
[master_port, client_port, master_port, client_port]):
addr = i.get_address()
self.assertEqual(addr, ('127.0.0.1', j))
for i in [self.site, self.site2]:
self.assertEqual(
i.get_address(),
self.dbenvMaster.repmgr_site_by_eid(
i.get_eid()).get_address())
for i in [self.site3, self.site4]:
self.assertEqual(
i.get_address(),
self.dbenvClient.repmgr_site_by_eid(
i.get_eid()).get_address())
else:
self.dbenvMaster.repmgr_set_local_site('127.0.0.1', master_port)
self.dbenvClient.repmgr_set_local_site('127.0.0.1', client_port)
self.dbenvMaster.repmgr_add_remote_site('127.0.0.1', client_port)
self.dbenvClient.repmgr_add_remote_site('127.0.0.1', master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321)
self.assertEqual(
self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY),
100123)
self.assertEqual(
self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY),
100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEqual(
self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT),
100234)
self.assertEqual(
self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT),
100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEqual(
self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345)
self.assertEqual(
self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER)
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT)
self.assertEqual(self.dbenvMaster.rep_get_nsites(), 2)
self.assertEqual(self.dbenvClient.rep_get_nsites(), 2)
self.assertEqual(self.dbenvMaster.rep_get_priority(), 10)
self.assertEqual(self.dbenvClient.rep_get_priority(), 0)
self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
import time
timeout = time.time() + 60
while time.time() < timeout and not (self.confirmed_master
and self.client_startupdone):
time.sleep(0.02)
if time.time() > timeout:
import sys
print >> sys.stderr, 'XXX: timeout happened beforestartup was confirmed - see issue 3892'
startup_timeout = True
d = self.dbenvMaster.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0]
self.assertEqual(d[0], '127.0.0.1')
self.assertEqual(d[1], client_port)
self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED
or d[2] == db.DB_REPMGR_DISCONNECTED)
d = self.dbenvClient.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0]
self.assertEqual(d[0], '127.0.0.1')
self.assertEqual(d[1], master_port)
self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED
or d[2] == db.DB_REPMGR_DISCONNECTED)
if db.version() >= (4, 6):
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR)
self.assertTrue('msgs_queued' in d)
self.dbMaster = db.DB(self.dbenvMaster)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.open('test', db.DB_HASH, db.DB_CREATE, 438, txn=txn)
txn.commit()
import time, os.path
timeout = time.time() + 10
while time.time() < timeout and not os.path.exists(
os.path.join(self.homeDirClient, 'test')):
time.sleep(0.01)
self.dbClient = db.DB(self.dbenvClient)
while True:
txn = self.dbenvClient.txn_begin()
try:
self.dbClient.open('test',
db.DB_HASH,
flags=db.DB_RDONLY,
mode=438,
txn=txn)
except db.DBRepHandleDeadError:
txn.abort()
self.dbClient.close()
self.dbClient = db.DB(self.dbenvClient)
continue
txn.commit()
break
txn = self.dbenvMaster.txn_begin()
self.dbMaster.put('ABC', '123', txn=txn)
txn.commit()
import time
timeout = time.time() + 10
v = None
while time.time() < timeout and v is None:
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get('ABC', txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
if time.time() >= timeout and startup_timeout:
self.skipTest(
'replication test skipped due to random failure, see issue 3892'
)
self.assertTrue(time.time() < timeout)
self.assertEqual('123', v)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.delete('ABC', txn=txn)
txn.commit()
timeout = time.time() + 10
while time.time() < timeout and v is not None:
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get('ABC', txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEqual(None, v)
return
class BasicTransactionTestCase(BasicTestCase):
if sys.version_info < (
2, 7) or sys.version_info >= (3, 0) and sys.version_info < (3, 2):
def assertIn(self, a, b, msg=None):
return self.assertTrue(a in b, msg=msg)
dbopenflags = db.DB_THREAD | db.DB_AUTO_COMMIT
useEnv = 1
envflags = db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_TXN
envsetflags = db.DB_AUTO_COMMIT
def tearDown(self):
self.txn.commit()
BasicTestCase.tearDown(self)
def populateDB(self):
txn = self.env.txn_begin()
BasicTestCase.populateDB(self, _txn=txn)
self.txn = self.env.txn_begin()
def test06_Transactions(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test06_Transactions...' % self.__class__.__name__
self.assertEqual(d.get('new rec', txn=self.txn), None)
d.put('new rec', 'this is a new record', self.txn)
self.assertEqual(d.get('new rec', txn=self.txn),
'this is a new record')
self.txn.abort()
self.assertEqual(d.get('new rec'), None)
self.txn = self.env.txn_begin()
self.assertEqual(d.get('new rec', txn=self.txn), None)
d.put('new rec', 'this is a new record', self.txn)
self.assertEqual(d.get('new rec', txn=self.txn),
'this is a new record')
self.txn.commit()
self.assertEqual(d.get('new rec'), 'this is a new record')
self.txn = self.env.txn_begin()
c = d.cursor(self.txn)
rec = c.first()
count = 0
while rec is not None:
count = count + 1
if verbose and count % 100 == 0:
print rec
rec = c.next()
self.assertEqual(count, self._numKeys + 1)
c.close()
self.txn.commit()
self.env.txn_checkpoint(0, 0, 0)
statDict = self.env.log_stat(0)
self.assertIn('magic', statDict)
self.assertIn('version', statDict)
self.assertIn('cur_file', statDict)
self.assertIn('region_nowait', statDict)
logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG)
self.assertNotEqual(logs, None)
for log in logs:
if verbose:
print 'log file: ' + log
logs = self.env.log_archive(db.DB_ARCH_REMOVE)
self.assertTrue(not logs)
self.txn = self.env.txn_begin()
return
if db.version() >= (4, 6):
def test08_exists(self):
txn = self.env.txn_begin()
self.d.put('abcde', 'ABCDE', txn=txn)
txn.commit()
txn = self.env.txn_begin()
self.assertTrue(
self.d.exists('abcde', txn=txn) == True,
'DB->exists() returns wrong value')
self.assertTrue(
self.d.exists('x', txn=txn) == False,
'DB->exists() returns wrong value')
txn.abort()
def test09_TxnTruncate(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test09_TxnTruncate...' % self.__class__.__name__
d.put('abcde', 'ABCDE')
txn = self.env.txn_begin()
num = d.truncate(txn)
self.assertTrue(num >= 1,
'truncate returned <= 0 on non-empty database')
num = d.truncate(txn)
self.assertEqual(
num, 0, 'truncate on empty DB returned nonzero (%r)' % (num, ))
txn.commit()
def test10_TxnLateUse(self):
txn = self.env.txn_begin()
txn.abort()
try:
txn.abort()
except db.DBError as e:
pass
else:
raise RuntimeError, 'DBTxn.abort() called after DB_TXN no longer valid w/o an exception'
txn = self.env.txn_begin()
txn.commit()
try:
txn.commit()
except db.DBError as e:
pass
else:
raise RuntimeError, 'DBTxn.commit() called after DB_TXN no longer valid w/o an exception'
if db.version() >= (4, 4):
def test_txn_name(self):
txn = self.env.txn_begin()
self.assertEqual(txn.get_name(), '')
txn.set_name('XXYY')
self.assertEqual(txn.get_name(), 'XXYY')
txn.set_name('')
self.assertEqual(txn.get_name(), '')
txn.abort()
def test_txn_set_timeout(self):
txn = self.env.txn_begin()
txn.set_timeout(1234567, db.DB_SET_LOCK_TIMEOUT)
txn.set_timeout(2345678, flags=db.DB_SET_TXN_TIMEOUT)
txn.abort()
def test_get_tx_max(self):
self.assertEqual(self.env.get_tx_max(), 30)
def test_get_tx_timestamp(self):
self.assertEqual(self.env.get_tx_timestamp(), self._t)
class DBEnv_general(DBEnv):
def test_get_open_flags(self):
flags = db.DB_CREATE | db.DB_INIT_MPOOL
self.env.open(self.homeDir, flags)
self.assertEqual(flags, self.env.get_open_flags())
def test_get_open_flags2(self):
flags = db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_THREAD
self.env.open(self.homeDir, flags)
self.assertEqual(flags, self.env.get_open_flags())
if db.version() >= (4, 7):
def test_lk_partitions(self):
for i in [10, 20, 40]:
self.env.set_lk_partitions(i)
self.assertEqual(i, self.env.get_lk_partitions())
def test_getset_intermediate_dir_mode(self):
self.assertEqual(None, self.env.get_intermediate_dir_mode())
for mode in ['rwx------', 'rw-rw-rw-', 'rw-r--r--']:
self.env.set_intermediate_dir_mode(mode)
self.assertEqual(mode, self.env.get_intermediate_dir_mode())
self.assertRaises(db.DBInvalidArgError,
self.env.set_intermediate_dir_mode, 'abcde')
return
if db.version() >= (4, 6):
def test_thread(self):
for i in [16, 100, 1000]:
self.env.set_thread_count(i)
self.assertEqual(i, self.env.get_thread_count())
def test_cache_max(self):
for size in [64, 128]:
size = size * 1024 * 1024
self.env.set_cache_max(0, size)
size2 = self.env.get_cache_max()
self.assertEqual(0, size2[0])
self.assertTrue(size <= size2[1])
self.assertTrue(2 * size > size2[1])
if db.version() >= (4, 4):
def test_mutex_stat(self):
self.env.open(self.homeDir,
db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK)
stat = self.env.mutex_stat()
self.assertTrue('mutex_inuse_max' in stat)
def test_lg_filemode(self):
for i in [384, 432, 438]:
self.env.set_lg_filemode(i)
self.assertEqual(i, self.env.get_lg_filemode())
def test_mp_max_openfd(self):
for i in [17, 31, 42]:
self.env.set_mp_max_openfd(i)
self.assertEqual(i, self.env.get_mp_max_openfd())
def test_mp_max_write(self):
for i in [100, 200, 300]:
for j in [1, 2, 3]:
j *= 1000000
self.env.set_mp_max_write(i, j)
v = self.env.get_mp_max_write()
self.assertEqual((i, j), v)
def test_invalid_txn(self):
self.assertRaises(db.DBInvalidArgError, self.env.txn_begin)
def test_mp_mmapsize(self):
for i in [16, 32, 64]:
i *= 1048576
self.env.set_mp_mmapsize(i)
self.assertEqual(i, self.env.get_mp_mmapsize())
def test_tmp_dir(self):
for i in ['a', 'bb', 'ccc']:
self.env.set_tmp_dir(i)
self.assertEqual(i, self.env.get_tmp_dir())
def test_flags(self):
self.env.set_flags(db.DB_AUTO_COMMIT, 1)
self.assertEqual(db.DB_AUTO_COMMIT, self.env.get_flags())
self.env.set_flags(db.DB_TXN_NOSYNC, 1)
self.assertEqual(db.DB_AUTO_COMMIT | db.DB_TXN_NOSYNC,
self.env.get_flags())
self.env.set_flags(db.DB_AUTO_COMMIT, 0)
self.assertEqual(db.DB_TXN_NOSYNC, self.env.get_flags())
self.env.set_flags(db.DB_TXN_NOSYNC, 0)
self.assertEqual(0, self.env.get_flags())
def test_lk_max_objects(self):
for i in [1000, 2000, 3000]:
self.env.set_lk_max_objects(i)
self.assertEqual(i, self.env.get_lk_max_objects())
def test_lk_max_locks(self):
for i in [1000, 2000, 3000]:
self.env.set_lk_max_locks(i)
self.assertEqual(i, self.env.get_lk_max_locks())
def test_lk_max_lockers(self):
for i in [1000, 2000, 3000]:
self.env.set_lk_max_lockers(i)
self.assertEqual(i, self.env.get_lk_max_lockers())
def test_lg_regionmax(self):
for i in [128, 256, 1000]:
i = i * 1024 * 1024
self.env.set_lg_regionmax(i)
j = self.env.get_lg_regionmax()
self.assertTrue(i <= j)
self.assertTrue(2 * i > j)
def test_lk_detect(self):
flags = [
db.DB_LOCK_DEFAULT, db.DB_LOCK_EXPIRE, db.DB_LOCK_MAXLOCKS,
db.DB_LOCK_MINLOCKS, db.DB_LOCK_MINWRITE, db.DB_LOCK_OLDEST,
db.DB_LOCK_RANDOM, db.DB_LOCK_YOUNGEST
]
flags.append(db.DB_LOCK_MAXWRITE)
for i in flags:
self.env.set_lk_detect(i)
self.assertEqual(i, self.env.get_lk_detect())
def test_lg_dir(self):
for i in ['a', 'bb', 'ccc', 'dddd']:
self.env.set_lg_dir(i)
self.assertEqual(i, self.env.get_lg_dir())
def test_lg_bsize(self):
log_size = 71680
self.env.set_lg_bsize(log_size)
self.assertTrue(self.env.get_lg_bsize() >= log_size)
self.assertTrue(self.env.get_lg_bsize() < 4 * log_size)
self.env.set_lg_bsize(4 * log_size)
self.assertTrue(self.env.get_lg_bsize() >= 4 * log_size)
def test_setget_data_dirs(self):
dirs = ('a', 'b', 'c', 'd')
for i in dirs:
self.env.set_data_dir(i)
self.assertEqual(dirs, self.env.get_data_dirs())
def test_setget_cachesize(self):
cachesize = (0, 536870912, 3)
self.env.set_cachesize(*cachesize)
self.assertEqual(cachesize, self.env.get_cachesize())
cachesize = (0, 1048576, 5)
self.env.set_cachesize(*cachesize)
cachesize2 = self.env.get_cachesize()
self.assertEqual(cachesize[0], cachesize2[0])
self.assertEqual(cachesize[2], cachesize2[2])
self.assertEqual(125, int(100.0 * cachesize2[1] / cachesize[1]))
self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL)
cachesize = (0, 2097152, 1)
self.assertRaises(db.DBInvalidArgError, self.env.set_cachesize,
*cachesize)
cachesize3 = self.env.get_cachesize()
self.assertEqual(cachesize2[0], cachesize3[0])
self.assertEqual(cachesize2[2], cachesize3[2])
self.assertTrue(cachesize2[1] <= cachesize3[1])
def test_set_cachesize_dbenv_db(self):
d = db.DB(self.env)
self.assertRaises(db.DBInvalidArgError, d.set_cachesize, 0,
1048576, 1)
def test_setget_shm_key(self):
shm_key = 137
self.env.set_shm_key(shm_key)
self.assertEqual(shm_key, self.env.get_shm_key())
self.env.set_shm_key(shm_key + 1)
self.assertEqual(shm_key + 1, self.env.get_shm_key())
self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL)
self.assertRaises(db.DBInvalidArgError, self.env.set_shm_key,
shm_key)
self.assertEqual(shm_key + 1, self.env.get_shm_key())
if db.version() >= (4, 4):
def test_mutex_setget_max(self):
v = self.env.mutex_get_max()
v2 = v * 2 + 1
self.env.mutex_set_max(v2)
self.assertEqual(v2, self.env.mutex_get_max())
self.env.mutex_set_max(v)
self.assertEqual(v, self.env.mutex_get_max())
self.env.open(self.homeDir, db.DB_CREATE)
self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_max, v2)
def test_mutex_setget_increment(self):
v = self.env.mutex_get_increment()
v2 = 127
self.env.mutex_set_increment(v2)
self.assertEqual(v2, self.env.mutex_get_increment())
self.env.mutex_set_increment(v)
self.assertEqual(v, self.env.mutex_get_increment())
self.env.open(self.homeDir, db.DB_CREATE)
self.assertRaises(db.DBInvalidArgError,
self.env.mutex_set_increment, v2)
def test_mutex_setget_tas_spins(self):
self.env.mutex_set_tas_spins(0)
v = self.env.mutex_get_tas_spins()
v2 = v * 2 + 1
self.env.mutex_set_tas_spins(v2)
self.assertEqual(v2, self.env.mutex_get_tas_spins())
self.env.mutex_set_tas_spins(v)
self.assertEqual(v, self.env.mutex_get_tas_spins())
self.env.open(self.homeDir, db.DB_CREATE)
self.env.mutex_set_tas_spins(v2)
def test_mutex_setget_align(self):
v = self.env.mutex_get_align()
v2 = 64
if v == 64:
v2 = 128
self.env.mutex_set_align(v2)
self.assertEqual(v2, self.env.mutex_get_align())
self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_align,
0)
self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_align,
17)
self.env.mutex_set_align(2 * v2)
self.assertEqual(2 * v2, self.env.mutex_get_align())
self.env.open(self.homeDir, db.DB_CREATE)
self.assertRaises(db.DBInvalidArgError, self.env.mutex_set_align,
v2)
class LockingTestCase(unittest.TestCase):
import sys
if sys.version_info[:3] < (2, 4, 0):
def assertTrue(self, expr, msg=None):
self.failUnless(expr,msg=msg)
def setUp(self):
self.homeDir = get_new_environment_path()
self.env = db.DBEnv()
self.env.open(self.homeDir, db.DB_THREAD | db.DB_INIT_MPOOL |
db.DB_INIT_LOCK | db.DB_CREATE)
def tearDown(self):
self.env.close()
test_support.rmtree(self.homeDir)
def test01_simple(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test01_simple..." % self.__class__.__name__
anID = self.env.lock_id()
if verbose:
print "locker ID: %s" % anID
lock = self.env.lock_get(anID, "some locked thing", db.DB_LOCK_WRITE)
if verbose:
print "Aquired lock: %s" % lock
self.env.lock_put(lock)
if verbose:
print "Released lock: %s" % lock
self.env.lock_id_free(anID)
def test02_threaded(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test02_threaded..." % self.__class__.__name__
threads = []
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_WRITE,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_READ,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_READ,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_WRITE,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_READ,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_READ,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_WRITE,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_WRITE,)))
threads.append(Thread(target = self.theThread,
args=(db.DB_LOCK_WRITE,)))
for t in threads:
import sys
if sys.version_info[0] < 3 :
t.setDaemon(True)
else :
t.daemon = True
t.start()
for t in threads:
t.join()
if db.version() >= (4, 2) :
def test03_lock_timeout(self):
self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT)
self.assertEqual(self.env.get_timeout(db.DB_SET_LOCK_TIMEOUT), 0)
self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT)
self.assertEqual(self.env.get_timeout(db.DB_SET_TXN_TIMEOUT), 0)
self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT)
self.assertEqual(self.env.get_timeout(db.DB_SET_LOCK_TIMEOUT), 123456)
self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT)
self.assertEqual(self.env.get_timeout(db.DB_SET_TXN_TIMEOUT), 7890123)
def test04_lock_timeout2(self):
self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT)
self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT)
self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT)
self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT)
def deadlock_detection() :
while not deadlock_detection.end :
deadlock_detection.count = \
self.env.lock_detect(db.DB_LOCK_EXPIRE)
if deadlock_detection.count :
while not deadlock_detection.end :
pass
break
time.sleep(0.01)
deadlock_detection.end=False
deadlock_detection.count=0
t=Thread(target=deadlock_detection)
import sys
if sys.version_info[0] < 3 :
t.setDaemon(True)
else :
t.daemon = True
t.start()
self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT)
anID = self.env.lock_id()
anID2 = self.env.lock_id()
self.assertNotEqual(anID, anID2)
lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE)
start_time=time.time()
self.assertRaises(db.DBLockNotGrantedError,
self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ)
end_time=time.time()
deadlock_detection.end=True
self.assertTrue((end_time-start_time) >= 0.1)
self.env.lock_put(lock)
t.join()
self.env.lock_id_free(anID)
self.env.lock_id_free(anID2)
if db.version() >= (4,6):
self.assertTrue(deadlock_detection.count>0)
def theThread(self, lockType):
import sys
if sys.version_info[0] < 3 :
name = currentThread().getName()
else :
name = currentThread().name
if lockType == db.DB_LOCK_WRITE:
lt = "write"
else:
lt = "read"
anID = self.env.lock_id()
if verbose:
print "%s: locker ID: %s" % (name, anID)
for i in xrange(1000) :
lock = self.env.lock_get(anID, "some locked thing", lockType)
if verbose:
print "%s: Aquired %s lock: %s" % (name, lt, lock)
self.env.lock_put(lock)
if verbose:
print "%s: Released %s lock: %s" % (name, lt, lock)
self.env.lock_id_free(anID)
def test01_basic_replication(self) :
master_port = test_support.find_unused_port()
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
client_port = test_support.find_unused_port()
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321)
self.assertEquals(self.dbenvMaster.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100123)
self.assertEquals(self.dbenvClient.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEquals(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100234)
self.assertEquals(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEquals(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100345)
self.assertEquals(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER);
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT);
self.assertEquals(self.dbenvMaster.rep_get_nsites(),2)
self.assertEquals(self.dbenvClient.rep_get_nsites(),2)
self.assertEquals(self.dbenvMaster.rep_get_priority(),10)
self.assertEquals(self.dbenvClient.rep_get_priority(),0)
self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time()+10
while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
d = self.dbenvMaster.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], client_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], master_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
if db.version() >= (4,6) :
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
self.assertTrue("msgs_queued" in d)
self.dbMaster=db.DB(self.dbenvMaster)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time,os.path
timeout=time.time()+10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient=db.DB(self.dbenvClient)
while True :
txn=self.dbenvClient.txn_begin()
try :
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY,
mode=0666, txn=txn)
except db.DBRepHandleDeadError :
txn.abort()
self.dbClient.close()
self.dbClient=db.DB(self.dbenvClient)
continue
txn.commit()
break
txn=self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout=time.time()+10
v=None
while (time.time()<timeout) and (v==None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals("123", v)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout=time.time()+10
while (time.time()<timeout) and (v!=None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals(None, v)
def test03c_SimpleCursorGetReturnsNone2(self):
# same test but raise exceptions instead of returning None
if verbose:
print '\n', '-=' * 30
print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \
self.__class__.__name__
old = self.d.set_get_returns_none(1)
self.assertEqual(old, 2)
old = self.d.set_get_returns_none(2)
self.assertEqual(old, 1)
self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0)
if db.version() >= (4, 6):
def test03d_SimpleCursorPriority(self) :
c = self.d.cursor()
c.set_priority(db.DB_PRIORITY_VERY_LOW) # Positional
self.assertEqual(db.DB_PRIORITY_VERY_LOW, c.get_priority())
c.set_priority(priority=db.DB_PRIORITY_HIGH) # Keyword
self.assertEqual(db.DB_PRIORITY_HIGH, c.get_priority())
c.close()
#----------------------------------------
def test04_PartialGetAndPut(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print "Running %s.test04_PartialGetAndPut..." % \
def test01_basic_replication(self) :
master_port = test_support.find_unused_port()
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
client_port = test_support.find_unused_port()
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321)
self.assertEquals(self.dbenvMaster.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100123)
self.assertEquals(self.dbenvClient.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEquals(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100234)
self.assertEquals(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEquals(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100345)
self.assertEquals(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER);
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT);
self.assertEquals(self.dbenvMaster.rep_get_nsites(),2)
self.assertEquals(self.dbenvClient.rep_get_nsites(),2)
self.assertEquals(self.dbenvMaster.rep_get_priority(),10)
self.assertEquals(self.dbenvClient.rep_get_priority(),0)
self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time()+10
while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) :
time.sleep(0.02)
# this fails on Windows as self.client_startupdone never gets set
# to True - see bug 3892. BUT - even though this assertion
# fails on Windows the rest of the test passes - so to prove
# that we let the rest of the test run. Sadly we can't
# make use of raising TestSkipped() here (unittest still
# reports it as an error), so we yell to stderr.
import sys
if sys.platform=="win32":
print >> sys.stderr, \
"XXX - windows bsddb replication fails on windows and is skipped"
print >> sys.stderr, "XXX - Please see issue #3892"
else:
self.assertTrue(time.time()<timeout)
d = self.dbenvMaster.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], client_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], master_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
if db.version() >= (4,6) :
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
self.assertTrue("msgs_queued" in d)
self.dbMaster=db.DB(self.dbenvMaster)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time,os.path
timeout=time.time()+10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient=db.DB(self.dbenvClient)
while True :
txn=self.dbenvClient.txn_begin()
try :
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY,
mode=0666, txn=txn)
except db.DBRepHandleDeadError :
txn.abort()
self.dbClient.close()
self.dbClient=db.DB(self.dbenvClient)
continue
txn.commit()
break
txn=self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout=time.time()+10
v=None
while (time.time()<timeout) and (v==None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals("123", v)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout=time.time()+10
while (time.time()<timeout) and (v!=None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals(None, v)
def test_suite():
suite = unittest.TestSuite()
if db.version() >= (4, 4):
suite.addTest(unittest.makeSuite(FileidResetTestCase))
return suite
def test01_basic_replication(self) :
master_port = test_support.find_unused_port()
client_port = test_support.find_unused_port()
if db.version() >= (5, 2) :
self.site = self.dbenvMaster.repmgr_site("127.0.0.1", master_port)
self.site.set_config(db.DB_GROUP_CREATOR, True)
self.site.set_config(db.DB_LOCAL_SITE, True)
self.site2 = self.dbenvMaster.repmgr_site("127.0.0.1", client_port)
self.site3 = self.dbenvClient.repmgr_site("127.0.0.1", master_port)
self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True)
self.site4 = self.dbenvClient.repmgr_site("127.0.0.1", client_port)
self.site4.set_config(db.DB_LOCAL_SITE, True)
d = {
db.DB_BOOTSTRAP_HELPER: [False, False, True, False],
db.DB_GROUP_CREATOR: [True, False, False, False],
db.DB_LEGACY: [False, False, False, False],
db.DB_LOCAL_SITE: [True, False, False, True],
db.DB_REPMGR_PEER: [False, False, False, False ],
}
for i, j in d.items() :
for k, v in \
zip([self.site, self.site2, self.site3, self.site4], j) :
if v :
self.assertTrue(k.get_config(i))
else :
self.assertFalse(k.get_config(i))
self.assertNotEqual(self.site.get_eid(), self.site2.get_eid())
self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid())
for i, j in zip([self.site, self.site2, self.site3, self.site4], \
[master_port, client_port, master_port, client_port]) :
addr = i.get_address()
self.assertEqual(addr, ("127.0.0.1", j))
for i in [self.site, self.site2] :
self.assertEqual(i.get_address(),
self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address())
for i in [self.site3, self.site4] :
self.assertEqual(i.get_address(),
self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address())
else :
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY,100321)
self.assertEqual(self.dbenvMaster.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100123)
self.assertEqual(self.dbenvClient.rep_get_timeout(
db.DB_REP_CONNECTION_RETRY), 100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEqual(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100234)
self.assertEqual(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_TIMEOUT), 100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEqual(self.dbenvMaster.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100345)
self.assertEqual(self.dbenvClient.rep_get_timeout(
db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER);
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT);
self.assertEqual(self.dbenvMaster.rep_get_nsites(),2)
self.assertEqual(self.dbenvClient.rep_get_nsites(),2)
self.assertEqual(self.dbenvMaster.rep_get_priority(),10)
self.assertEqual(self.dbenvClient.rep_get_priority(),0)
self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time()+10
while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
d = self.dbenvMaster.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0] # There is only one
self.assertEqual(d[0], "127.0.0.1")
self.assertEqual(d[1], client_port)
self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
(d[2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0] # There is only one
self.assertEqual(d[0], "127.0.0.1")
self.assertEqual(d[1], master_port)
self.assertTrue((d[2]==db.DB_REPMGR_CONNECTED) or \
(d[2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR);
self.assertTrue("msgs_queued" in d)
self.dbMaster=db.DB(self.dbenvMaster)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time,os.path
timeout=time.time()+10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient=db.DB(self.dbenvClient)
while True :
txn=self.dbenvClient.txn_begin()
try :
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY,
mode=0666, txn=txn)
except db.DBRepHandleDeadError :
txn.abort()
self.dbClient.close()
self.dbClient=db.DB(self.dbenvClient)
continue
txn.commit()
break
txn=self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout=time.time()+10
v=None
while (time.time()<timeout) and (v is None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v is None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEqual("123", v)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout=time.time()+10
while (time.time()<timeout) and (v is not None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v is None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEqual(None, v)
def test_suite():
suite = unittest.TestSuite()
if db.version() >= (4, 3):
suite.addTest(unittest.makeSuite(DBSequenceTest))
return suite
class DBSequenceTest(unittest.TestCase):
def setUp(self):
self.int_32_max = 0x100000000
self.homeDir = get_new_environment_path()
self.filename = "test"
self.dbenv = db.DBEnv()
self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL, 0666)
self.d = db.DB(self.dbenv)
self.d.open(self.filename, db.DB_BTREE, db.DB_CREATE, 0666)
def tearDown(self):
if hasattr(self, 'seq'):
self.seq.close()
del self.seq
if hasattr(self, 'd'):
self.d.close()
del self.d
if hasattr(self, 'dbenv'):
self.dbenv.close()
del self.dbenv
test_support.rmtree(self.homeDir)
def test_get(self):
self.seq = db.DBSequence(self.d, flags=0)
start_value = 10 * self.int_32_max
self.assertEqual(0xA00000000, start_value)
self.assertEqual(None, self.seq.initial_value(start_value))
self.assertEqual(None,
self.seq.open(key='id', txn=None, flags=db.DB_CREATE))
self.assertEqual(start_value, self.seq.get(5))
self.assertEqual(start_value + 5, self.seq.get())
def test_remove(self):
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(
None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(None, self.seq.remove(txn=None, flags=0))
del self.seq
def test_get_key(self):
self.seq = db.DBSequence(self.d, flags=0)
key = 'foo'
self.assertEqual(None,
self.seq.open(key=key, txn=None, flags=db.DB_CREATE))
self.assertEqual(key, self.seq.get_key())
def test_get_dbp(self):
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(
None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(self.d, self.seq.get_dbp())
def test_cachesize(self):
self.seq = db.DBSequence(self.d, flags=0)
cashe_size = 10
self.assertEqual(None, self.seq.set_cachesize(cashe_size))
self.assertEqual(
None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(cashe_size, self.seq.get_cachesize())
def test_flags(self):
self.seq = db.DBSequence(self.d, flags=0)
flag = db.DB_SEQ_WRAP
self.assertEqual(None, self.seq.set_flags(flag))
self.assertEqual(
None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(flag, self.seq.get_flags() & flag)
def test_range(self):
self.seq = db.DBSequence(self.d, flags=0)
seq_range = (10 * self.int_32_max, 11 * self.int_32_max - 1)
self.assertEqual(None, self.seq.set_range(seq_range))
self.seq.initial_value(seq_range[0])
self.assertEqual(
None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(seq_range, self.seq.get_range())
def test_stat(self):
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(
None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
stat = self.seq.stat()
for param in ('nowait', 'min', 'max', 'value', 'current', 'flags',
'cache_size', 'last_value', 'wait'):
self.assertTrue(param in stat,
"parameter %s isn't in stat info" % param)
if db.version() >= (4, 7):
# This code checks a crash solved in Berkeley DB 4.7
def test_stat_crash(self):
d = db.DB()
d.open(None, dbtype=db.DB_HASH, flags=db.DB_CREATE) # In RAM
seq = db.DBSequence(d, flags=0)
self.assertRaises(db.DBNotFoundError,
seq.open,
key='id',
txn=None,
flags=0)
self.assertRaises(db.DBInvalidArgError, seq.stat)
d.close()
def test_64bits(self):
# We don't use both extremes because they are problematic
value_plus = (1L << 63) - 2
self.assertEqual(9223372036854775806L, value_plus)
value_minus = (-1L << 63) + 1 # Two complement
self.assertEqual(-9223372036854775807L, value_minus)
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.initial_value(value_plus - 1))
self.assertEqual(None,
self.seq.open(key='id', txn=None, flags=db.DB_CREATE))
self.assertEqual(value_plus - 1, self.seq.get(1))
self.assertEqual(value_plus, self.seq.get(1))
self.seq.remove(txn=None, flags=0)
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.initial_value(value_minus))
self.assertEqual(None,
self.seq.open(key='id', txn=None, flags=db.DB_CREATE))
self.assertEqual(value_minus, self.seq.get(1))
self.assertEqual(value_minus + 1, self.seq.get(1))
def test_multiple_close(self):
self.seq = db.DBSequence(self.d)
self.seq.close() # You can close a Sequence multiple times
self.seq.close()
self.seq.close()
class DBBaseReplication(DBReplicationManager):
def setUp(self) :
DBReplicationManager.setUp(self)
def confirmed_master(a,b,c) :
if (b == db.DB_EVENT_REP_MASTER) or (b == db.DB_EVENT_REP_ELECTED) :
self.confirmed_master = True
def client_startupdone(a,b,c) :
if b == db.DB_EVENT_REP_STARTUPDONE :
self.client_startupdone = True
self.dbenvMaster.set_event_notify(confirmed_master)
self.dbenvClient.set_event_notify(client_startupdone)
import Queue
self.m2c = Queue.Queue()
self.c2m = Queue.Queue()
# There are only two nodes, so we don't need to
# do any routing decision
def m2c(dbenv, control, rec, lsnp, envid, flags) :
self.m2c.put((control, rec))
def c2m(dbenv, control, rec, lsnp, envid, flags) :
self.c2m.put((control, rec))
self.dbenvMaster.rep_set_transport(13,m2c)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_transport(3,c2m)
self.dbenvClient.rep_set_priority(0)
self.assertEquals(self.dbenvMaster.rep_get_priority(),10)
self.assertEquals(self.dbenvClient.rep_get_priority(),0)
#self.dbenvMaster.set_verbose(db.DB_VERB_REPLICATION, True)
#self.dbenvMaster.set_verbose(db.DB_VERB_FILEOPS_ALL, True)
#self.dbenvClient.set_verbose(db.DB_VERB_REPLICATION, True)
#self.dbenvClient.set_verbose(db.DB_VERB_FILEOPS_ALL, True)
def thread_master() :
return self.thread_do(self.dbenvMaster, self.c2m, 3,
self.master_doing_election, True)
def thread_client() :
return self.thread_do(self.dbenvClient, self.m2c, 13,
self.client_doing_election, False)
from threading import Thread
t_m=Thread(target=thread_master)
t_c=Thread(target=thread_client)
import sys
if sys.version_info[0] < 3 :
t_m.setDaemon(True)
t_c.setDaemon(True)
else :
t_m.daemon = True
t_c.daemon = True
self.t_m = t_m
self.t_c = t_c
self.dbMaster = self.dbClient = None
self.master_doing_election=[False]
self.client_doing_election=[False]
def tearDown(self):
if self.dbClient :
self.dbClient.close()
if self.dbMaster :
self.dbMaster.close()
self.m2c.put(None)
self.c2m.put(None)
self.t_m.join()
self.t_c.join()
self.dbenvClient.close()
self.dbenvMaster.close()
test_support.rmtree(self.homeDirClient)
test_support.rmtree(self.homeDirMaster)
def basic_rep_threading(self) :
self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER)
self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT)
def thread_do(env, q, envid, election_status, must_be_master) :
while True :
v=q.get()
if v == None : return
env.rep_process_message(v[0], v[1], envid)
self.thread_do = thread_do
self.t_m.start()
self.t_c.start()
def test01_basic_replication(self) :
self.basic_rep_threading()
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time()+10
while (time.time()<timeout) and not (self.confirmed_master and
self.client_startupdone) :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.dbMaster=db.DB(self.dbenvMaster)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time,os.path
timeout=time.time()+10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient=db.DB(self.dbenvClient)
while True :
txn=self.dbenvClient.txn_begin()
try :
self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY,
mode=0666, txn=txn)
except db.DBRepHandleDeadError :
txn.abort()
self.dbClient.close()
self.dbClient=db.DB(self.dbenvClient)
continue
txn.commit()
break
d = self.dbenvMaster.rep_stat(flags=db.DB_STAT_CLEAR);
self.assertTrue("master_changes" in d)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout=time.time()+10
v=None
while (time.time()<timeout) and (v==None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals("123", v)
txn=self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout=time.time()+10
while (time.time()<timeout) and (v!=None) :
txn=self.dbenvClient.txn_begin()
v=self.dbClient.get("ABC", txn=txn)
txn.commit()
if v==None :
time.sleep(0.02)
self.assertTrue(time.time()<timeout)
self.assertEquals(None, v)
if db.version() >= (4,7) :
def test02_test_request(self) :
self.basic_rep_threading()
(minimum, maximum) = self.dbenvClient.rep_get_request()
self.dbenvClient.rep_set_request(minimum-1, maximum+1)
self.assertEqual(self.dbenvClient.rep_get_request(),
(minimum-1, maximum+1))
if db.version() >= (4,6) :
def test03_master_election(self) :
# Get ready to hold an election
#self.dbenvMaster.rep_start(flags=db.DB_REP_MASTER)
self.dbenvMaster.rep_start(flags=db.DB_REP_CLIENT)
self.dbenvClient.rep_start(flags=db.DB_REP_CLIENT)
def thread_do(env, q, envid, election_status, must_be_master) :
while True :
v=q.get()
if v == None : return
r = env.rep_process_message(v[0],v[1],envid)
if must_be_master and self.confirmed_master :
self.dbenvMaster.rep_start(flags = db.DB_REP_MASTER)
must_be_master = False
if r[0] == db.DB_REP_HOLDELECTION :
def elect() :
while True :
try :
env.rep_elect(2, 1)
election_status[0] = False
break
except db.DBRepUnavailError :
pass
if not election_status[0] and not self.confirmed_master :
from threading import Thread
election_status[0] = True
t=Thread(target=elect)
import sys
if sys.version_info[0] < 3 :
t.setDaemon(True)
else :
t.daemon = True
t.start()
self.thread_do = thread_do
self.t_m.start()
self.t_c.start()
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 50000)
self.client_doing_election[0] = True
while True :
try :
self.dbenvClient.rep_elect(2, 1)
self.client_doing_election[0] = False
break
except db.DBRepUnavailError :
pass
self.assertTrue(self.confirmed_master)
if db.version() >= (4,7) :
def test04_test_clockskew(self) :
fast, slow = 1234, 1230
self.dbenvMaster.rep_set_clockskew(fast, slow)
self.assertEqual((fast, slow),
self.dbenvMaster.rep_get_clockskew())
self.basic_rep_threading()
def _destroy_env(self):
if self.nosync or db.version()[:2] == (4, 6):
self.dbenv.log_flush()
self.db.close()
self.dbenv.close()
class BasicTestCase(unittest.TestCase):
dbtype = db.DB_UNKNOWN
cachesize = (0, 1048576, 1)
dbopenflags = 0
dbsetflags = 0
dbmode = 432
dbname = None
useEnv = 0
envflags = 0
envsetflags = 0
_numKeys = 1002
def setUp(self):
if self.useEnv:
self.homeDir = get_new_environment_path()
try:
self.env = db.DBEnv()
self.env.set_lg_max(1048576)
self.env.set_tx_max(30)
self._t = int(time.time())
self.env.set_tx_timestamp(self._t)
self.env.set_flags(self.envsetflags, 1)
self.env.open(self.homeDir, self.envflags | db.DB_CREATE)
self.filename = 'test'
except:
test_support.rmtree(self.homeDir)
raise
else:
self.env = None
self.filename = get_new_database_path()
self.d = db.DB(self.env)
if not self.useEnv:
self.d.set_cachesize(*self.cachesize)
cachesize = self.d.get_cachesize()
self.assertEqual(cachesize[0], self.cachesize[0])
self.assertEqual(cachesize[2], self.cachesize[2])
self.assertEqual(125,
int(100.0 * cachesize[1] / self.cachesize[1]))
self.d.set_flags(self.dbsetflags)
if self.dbname:
self.d.open(self.filename, self.dbname, self.dbtype,
self.dbopenflags | db.DB_CREATE, self.dbmode)
else:
self.d.open(self.filename,
mode=self.dbmode,
dbtype=self.dbtype,
flags=self.dbopenflags | db.DB_CREATE)
if not self.useEnv:
self.assertRaises(db.DBInvalidArgError, self.d.set_cachesize,
*self.cachesize)
self.populateDB()
return
def tearDown(self):
self.d.close()
if self.env is not None:
self.env.close()
test_support.rmtree(self.homeDir)
else:
os.remove(self.filename)
return
def populateDB(self, _txn=None):
d = self.d
for x in range(self._numKeys // 2):
key = '%04d' % (self._numKeys - x)
data = self.makeData(key)
d.put(key, data, _txn)
d.put('empty value', '', _txn)
for x in range(self._numKeys // 2 - 1):
key = '%04d' % x
data = self.makeData(key)
d.put(key, data, _txn)
if _txn:
_txn.commit()
num = len(d)
if verbose:
print 'created %d records' % num
def makeData(self, key):
return DASH.join([key] * 5)
def test01_GetsAndPuts(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test01_GetsAndPuts...' % self.__class__.__name__
for key in ['0001', '0100', '0400', '0700', '0999']:
data = d.get(key)
if verbose:
print data
self.assertEqual(d.get('0321'), '0321-0321-0321-0321-0321')
self.assertEqual(d.get('abcd'), None)
try:
d.delete('abcd')
except db.DBNotFoundError as val:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_NOTFOUND)
else:
self.assertEqual(val.args[0], db.DB_NOTFOUND)
if verbose:
print val
else:
self.fail('expected exception')
d.put('abcd', 'a new record')
self.assertEqual(d.get('abcd'), 'a new record')
d.put('abcd', 'same key')
if self.dbsetflags & db.DB_DUP:
self.assertEqual(d.get('abcd'), 'a new record')
else:
self.assertEqual(d.get('abcd'), 'same key')
try:
d.put('abcd', 'this should fail', flags=db.DB_NOOVERWRITE)
except db.DBKeyExistError as val:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_KEYEXIST)
else:
self.assertEqual(val.args[0], db.DB_KEYEXIST)
if verbose:
print val
else:
self.fail('expected exception')
if self.dbsetflags & db.DB_DUP:
self.assertEqual(d.get('abcd'), 'a new record')
else:
self.assertEqual(d.get('abcd'), 'same key')
d.sync()
d.close()
del d
self.d = db.DB(self.env)
if self.dbname:
self.d.open(self.filename, self.dbname)
else:
self.d.open(self.filename)
d = self.d
self.assertEqual(d.get('0321'), '0321-0321-0321-0321-0321')
if self.dbsetflags & db.DB_DUP:
self.assertEqual(d.get('abcd'), 'a new record')
else:
self.assertEqual(d.get('abcd'), 'same key')
rec = d.get_both('0555', '0555-0555-0555-0555-0555')
if verbose:
print rec
self.assertEqual(d.get_both('0555', 'bad data'), None)
data = d.get('bad key', 'bad data')
self.assertEqual(data, 'bad data')
data = d.get('bad key', self)
self.assertEqual(data, self)
s = d.stat()
self.assertEqual(type(s), type({}))
if verbose:
print 'd.stat() returned this dictionary:'
pprint(s)
return
def test02_DictionaryMethods(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test02_DictionaryMethods...' % self.__class__.__name__
for key in ['0002', '0101', '0401', '0701', '0998']:
data = d[key]
self.assertEqual(data, self.makeData(key))
if verbose:
print data
self.assertEqual(len(d), self._numKeys)
keys = d.keys()
self.assertEqual(len(keys), self._numKeys)
self.assertEqual(type(keys), type([]))
d['new record'] = 'a new record'
self.assertEqual(len(d), self._numKeys + 1)
keys = d.keys()
self.assertEqual(len(keys), self._numKeys + 1)
d['new record'] = 'a replacement record'
self.assertEqual(len(d), self._numKeys + 1)
keys = d.keys()
self.assertEqual(len(keys), self._numKeys + 1)
if verbose:
print 'the first 10 keys are:'
pprint(keys[:10])
self.assertEqual(d['new record'], 'a replacement record')
self.assertEqual(d.has_key('0001', None), 1)
self.assertEqual(d.has_key('spam', txn=None), 0)
items = d.items()
self.assertEqual(len(items), self._numKeys + 1)
self.assertEqual(type(items), type([]))
self.assertEqual(type(items[0]), type(()))
self.assertEqual(len(items[0]), 2)
if verbose:
print 'the first 10 items are:'
pprint(items[:10])
values = d.values()
self.assertEqual(len(values), self._numKeys + 1)
self.assertEqual(type(values), type([]))
if verbose:
print 'the first 10 values are:'
pprint(values[:10])
return
def test02b_SequenceMethods(self):
d = self.d
for key in ['0002', '0101', '0401', '0701', '0998']:
data = d[key]
self.assertEqual(data, self.makeData(key))
if verbose:
print data
self.assertTrue(hasattr(d, '__contains__'))
self.assertTrue('0401' in d)
self.assertFalse('1234' in d)
def test03_SimpleCursorStuff(self, get_raises_error=0, set_raises_error=0):
if verbose:
print '\n', '-=' * 30
print 'Running %s.test03_SimpleCursorStuff (get_error %s, set_error %s)...' % (
self.__class__.__name__, get_raises_error, set_raises_error)
if self.env and self.dbopenflags & db.DB_AUTO_COMMIT:
txn = self.env.txn_begin()
else:
txn = None
c = self.d.cursor(txn=txn)
rec = c.first()
count = 0
while rec is not None:
count = count + 1
if verbose and count % 100 == 0:
print rec
try:
rec = c.next()
except db.DBNotFoundError as val:
if get_raises_error:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_NOTFOUND)
else:
self.assertEqual(val.args[0], db.DB_NOTFOUND)
if verbose:
print val
rec = None
else:
self.fail('unexpected DBNotFoundError')
self.assertEqual(
c.get_current_size(), len(c.current()[1]),
'%s != len(%r)' % (c.get_current_size(), c.current()[1]))
self.assertEqual(count, self._numKeys)
rec = c.last()
count = 0
while rec is not None:
count = count + 1
if verbose and count % 100 == 0:
print rec
try:
rec = c.prev()
except db.DBNotFoundError as val:
if get_raises_error:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_NOTFOUND)
else:
self.assertEqual(val.args[0], db.DB_NOTFOUND)
if verbose:
print val
rec = None
else:
self.fail('unexpected DBNotFoundError')
self.assertEqual(count, self._numKeys)
rec = c.set('0505')
rec2 = c.current()
self.assertEqual(rec, rec2)
self.assertEqual(rec[0], '0505')
self.assertEqual(rec[1], self.makeData('0505'))
self.assertEqual(c.get_current_size(), len(rec[1]))
rec = c.set('empty value')
self.assertEqual(rec[1], '')
self.assertEqual(c.get_current_size(), 0)
try:
n = c.set('bad key')
except db.DBNotFoundError as val:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_NOTFOUND)
else:
self.assertEqual(val.args[0], db.DB_NOTFOUND)
if verbose:
print val
else:
if set_raises_error:
self.fail('expected exception')
if n is not None:
self.fail('expected None: %r' % (n, ))
rec = c.get_both('0404', self.makeData('0404'))
self.assertEqual(rec, ('0404', self.makeData('0404')))
try:
n = c.get_both('0404', 'bad data')
except db.DBNotFoundError as val:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_NOTFOUND)
else:
self.assertEqual(val.args[0], db.DB_NOTFOUND)
if verbose:
print val
else:
if get_raises_error:
self.fail('expected exception')
if n is not None:
self.fail('expected None: %r' % (n, ))
if self.d.get_type() == db.DB_BTREE:
rec = c.set_range('011')
if verbose:
print "searched for '011', found: ", rec
rec = c.set_range('011', dlen=0, doff=0)
if verbose:
print "searched (partial) for '011', found: ", rec
if rec[1] != '':
self.fail('expected empty data portion')
ev = c.set_range('empty value')
if verbose:
print "search for 'empty value' returned", ev
if ev[1] != '':
self.fail('empty value lookup failed')
c.set('0499')
c.delete()
try:
rec = c.current()
except db.DBKeyEmptyError as val:
if get_raises_error:
if sys.version_info < (2, 6):
self.assertEqual(val[0], db.DB_KEYEMPTY)
else:
self.assertEqual(val.args[0], db.DB_KEYEMPTY)
if verbose:
print val
else:
self.fail('unexpected DBKeyEmptyError')
else:
if get_raises_error:
self.fail('DBKeyEmptyError exception expected')
c.next()
c2 = c.dup(db.DB_POSITION)
self.assertEqual(c.current(), c2.current())
c2.put('', 'a new value', db.DB_CURRENT)
self.assertEqual(c.current(), c2.current())
self.assertEqual(c.current()[1], 'a new value')
c2.put('', 'er', db.DB_CURRENT, dlen=0, doff=5)
self.assertEqual(c2.current()[1], 'a newer value')
c.close()
c2.close()
if txn:
txn.commit()
methods_to_test = {
'current': (),
'delete': (),
'dup': (db.DB_POSITION, ),
'first': (),
'get': (0, ),
'next': (),
'prev': (),
'last': (),
'put': ('', 'spam', db.DB_CURRENT),
'set': ('0505', )
}
for method, args in methods_to_test.items():
try:
if verbose:
print "attempting to use a closed cursor's %s method" % method
getattr(c, method)(*args)
except db.DBError as val:
if sys.version_info < (2, 6):
self.assertEqual(val[0], 0)
else:
self.assertEqual(val.args[0], 0)
if verbose:
print val
else:
self.fail(
"no exception raised when using a buggy cursor's%s method"
% method)
oldcursor = self.d.cursor(txn=txn)
self.d.close()
del oldcursor
return
def test03b_SimpleCursorWithoutGetReturnsNone0(self):
if verbose:
print '\n', '-=' * 30
print 'Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone...' % self.__class__.__name__
old = self.d.set_get_returns_none(0)
self.assertEqual(old, 2)
self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1)
def test03b_SimpleCursorWithGetReturnsNone1(self):
if verbose:
print '\n', '-=' * 30
print 'Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone...' % self.__class__.__name__
old = self.d.set_get_returns_none(1)
self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=1)
def test03c_SimpleCursorGetReturnsNone2(self):
if verbose:
print '\n', '-=' * 30
print 'Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone...' % self.__class__.__name__
old = self.d.set_get_returns_none(1)
self.assertEqual(old, 2)
old = self.d.set_get_returns_none(2)
self.assertEqual(old, 1)
self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0)
if db.version() >= (4, 6):
def test03d_SimpleCursorPriority(self):
c = self.d.cursor()
c.set_priority(db.DB_PRIORITY_VERY_LOW)
self.assertEqual(db.DB_PRIORITY_VERY_LOW, c.get_priority())
c.set_priority(priority=db.DB_PRIORITY_HIGH)
self.assertEqual(db.DB_PRIORITY_HIGH, c.get_priority())
c.close()
def test04_PartialGetAndPut(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test04_PartialGetAndPut...' % self.__class__.__name__
key = 'partialTest'
data = '1' * 1000 + '2' * 1000
d.put(key, data)
self.assertEqual(d.get(key), data)
self.assertEqual(d.get(key, dlen=20, doff=990),
'1111111111' + '2222222222')
d.put('partialtest2', '1' * 30000 + 'robin')
self.assertEqual(d.get('partialtest2', dlen=5, doff=30000), 'robin')
if self.dbsetflags != db.DB_DUP:
d.put(key, '0000', dlen=2000, doff=0)
self.assertEqual(d.get(key), '0000')
d.put(key, '1111', dlen=1, doff=2)
self.assertEqual(d.get(key), '0011110')
def test05_GetSize(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test05_GetSize...' % self.__class__.__name__
for i in range(1, 50000, 500):
key = 'size%s' % i
d.put(key, '1' * i)
self.assertEqual(d.get_size(key), i)
def test06_Truncate(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print 'Running %s.test06_Truncate...' % self.__class__.__name__
d.put('abcde', 'ABCDE')
num = d.truncate()
self.assertTrue(num >= 1,
'truncate returned <= 0 on non-empty database')
num = d.truncate()
self.assertEqual(
num, 0, 'truncate on empty DB returned nonzero (%r)' % (num, ))
def test07_verify(self):
self.d.close()
d = db.DB(self.env)
d.verify(self.filename)
if db.version() >= (4, 6):
def test08_exists(self):
self.d.put('abcde', 'ABCDE')
self.assertTrue(
self.d.exists('abcde') == True,
'DB->exists() returns wrong value')
self.assertTrue(
self.d.exists('x') == False,
'DB->exists() returns wrong value')
if db.version() >= (4, 7):
def test_compact(self):
d = self.d
self.assertEqual(0, d.compact(flags=db.DB_FREELIST_ONLY))
self.assertEqual(0, d.compact(flags=db.DB_FREELIST_ONLY))
d.put('abcde', 'ABCDE')
d.put('bcde', 'BCDE')
d.put('abc', 'ABC')
d.put('monty', 'python')
d.delete('abc')
d.delete('bcde')
d.compact(start='abcde',
stop='monty',
txn=None,
compact_fillpercent=42,
compact_pages=1,
compact_timeout=50000000,
flags=db.DB_FREELIST_ONLY | db.DB_FREE_SPACE)
return
class BasicTransactionTestCase(BasicTestCase):
import sys
if sys.version_info < (2, 4):
def assertTrue(self, expr, msg=None):
return self.failUnless(expr, msg=msg)
if (sys.version_info < (2, 7)) or ((sys.version_info >=
(3, 0)) and (sys.version_info <
(3, 2))):
def assertIn(self, a, b, msg=None):
return self.assertTrue(a in b, msg=msg)
dbopenflags = db.DB_THREAD | db.DB_AUTO_COMMIT
useEnv = 1
envflags = (db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK
| db.DB_INIT_TXN)
envsetflags = db.DB_AUTO_COMMIT
def tearDown(self):
self.txn.commit()
BasicTestCase.tearDown(self)
def populateDB(self):
txn = self.env.txn_begin()
BasicTestCase.populateDB(self, _txn=txn)
self.txn = self.env.txn_begin()
def test06_Transactions(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print "Running %s.test06_Transactions..." % self.__class__.__name__
self.assertEqual(d.get('new rec', txn=self.txn), None)
d.put('new rec', 'this is a new record', self.txn)
self.assertEqual(d.get('new rec', txn=self.txn),
'this is a new record')
self.txn.abort()
self.assertEqual(d.get('new rec'), None)
self.txn = self.env.txn_begin()
self.assertEqual(d.get('new rec', txn=self.txn), None)
d.put('new rec', 'this is a new record', self.txn)
self.assertEqual(d.get('new rec', txn=self.txn),
'this is a new record')
self.txn.commit()
self.assertEqual(d.get('new rec'), 'this is a new record')
self.txn = self.env.txn_begin()
c = d.cursor(self.txn)
rec = c.first()
count = 0
while rec is not None:
count = count + 1
if verbose and count % 100 == 0:
print rec
rec = c.next()
self.assertEqual(count, self._numKeys + 1)
c.close() # Cursors *MUST* be closed before commit!
self.txn.commit()
# flush pending updates
self.env.txn_checkpoint(0, 0, 0)
statDict = self.env.log_stat(0)
self.assertIn('magic', statDict)
self.assertIn('version', statDict)
self.assertIn('cur_file', statDict)
self.assertIn('region_nowait', statDict)
# must have at least one log file present:
logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG)
self.assertNotEqual(logs, None)
for log in logs:
if verbose:
print 'log file: ' + log
if db.version() >= (4, 2):
logs = self.env.log_archive(db.DB_ARCH_REMOVE)
self.assertTrue(not logs)
self.txn = self.env.txn_begin()
#----------------------------------------
if db.version() >= (4, 6):
def test08_exists(self):
txn = self.env.txn_begin()
self.d.put("abcde", "ABCDE", txn=txn)
txn.commit()
txn = self.env.txn_begin()
self.assertTrue(
self.d.exists("abcde", txn=txn) == True,
"DB->exists() returns wrong value")
self.assertTrue(
self.d.exists("x", txn=txn) == False,
"DB->exists() returns wrong value")
txn.abort()
#----------------------------------------
def test09_TxnTruncate(self):
d = self.d
if verbose:
print '\n', '-=' * 30
print "Running %s.test09_TxnTruncate..." % self.__class__.__name__
d.put("abcde", "ABCDE")
txn = self.env.txn_begin()
num = d.truncate(txn)
self.assertTrue(num >= 1,
"truncate returned <= 0 on non-empty database")
num = d.truncate(txn)
self.assertEqual(
num, 0, "truncate on empty DB returned nonzero (%r)" % (num, ))
txn.commit()
#----------------------------------------
def test10_TxnLateUse(self):
txn = self.env.txn_begin()
txn.abort()
try:
txn.abort()
except db.DBError, e:
pass
else:
def _destroy_env(self):
if self.nosync or db.version()[:2] == (4, 6):
self.dbenv.log_flush()
self.db.close()
self.dbenv.close()
def test01_basic_replication(self):
master_port = test_support.find_unused_port()
client_port = test_support.find_unused_port()
if db.version() >= (5, 2):
self.site = self.dbenvMaster.repmgr_site('127.0.0.1', master_port)
self.site.set_config(db.DB_GROUP_CREATOR, True)
self.site.set_config(db.DB_LOCAL_SITE, True)
self.site2 = self.dbenvMaster.repmgr_site('127.0.0.1', client_port)
self.site3 = self.dbenvClient.repmgr_site('127.0.0.1', master_port)
self.site3.set_config(db.DB_BOOTSTRAP_HELPER, True)
self.site4 = self.dbenvClient.repmgr_site('127.0.0.1', client_port)
self.site4.set_config(db.DB_LOCAL_SITE, True)
d = {db.DB_BOOTSTRAP_HELPER: [False,
False,
True,
False],
db.DB_GROUP_CREATOR: [True,
False,
False,
False],
db.DB_LEGACY: [False,
False,
False,
False],
db.DB_LOCAL_SITE: [True,
False,
False,
True],
db.DB_REPMGR_PEER: [False,
False,
False,
False]}
for i, j in d.items():
for k, v in zip([self.site,
self.site2,
self.site3,
self.site4], j):
if v:
self.assertTrue(k.get_config(i))
else:
self.assertFalse(k.get_config(i))
self.assertNotEqual(self.site.get_eid(), self.site2.get_eid())
self.assertNotEqual(self.site3.get_eid(), self.site4.get_eid())
for i, j in zip([self.site,
self.site2,
self.site3,
self.site4], [master_port,
client_port,
master_port,
client_port]):
addr = i.get_address()
self.assertEqual(addr, ('127.0.0.1', j))
for i in [self.site, self.site2]:
self.assertEqual(i.get_address(), self.dbenvMaster.repmgr_site_by_eid(i.get_eid()).get_address())
for i in [self.site3, self.site4]:
self.assertEqual(i.get_address(), self.dbenvClient.repmgr_site_by_eid(i.get_eid()).get_address())
else:
self.dbenvMaster.repmgr_set_local_site('127.0.0.1', master_port)
self.dbenvClient.repmgr_set_local_site('127.0.0.1', client_port)
self.dbenvMaster.repmgr_add_remote_site('127.0.0.1', client_port)
self.dbenvClient.repmgr_add_remote_site('127.0.0.1', master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321)
self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100123)
self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY), 100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100234)
self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT), 100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEqual(self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345)
self.assertEqual(self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER)
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT)
self.assertEqual(self.dbenvMaster.rep_get_nsites(), 2)
self.assertEqual(self.dbenvClient.rep_get_nsites(), 2)
self.assertEqual(self.dbenvMaster.rep_get_priority(), 10)
self.assertEqual(self.dbenvClient.rep_get_priority(), 0)
self.assertEqual(self.dbenvMaster.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL)
self.assertEqual(self.dbenvClient.repmgr_get_ack_policy(), db.DB_REPMGR_ACKS_ALL)
import time
timeout = time.time() + 60
while time.time() < timeout and not (self.confirmed_master and self.client_startupdone):
time.sleep(0.02)
if time.time() > timeout:
import sys
print >> sys.stderr, 'XXX: timeout happened beforestartup was confirmed - see issue 3892'
startup_timeout = True
d = self.dbenvMaster.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0]
self.assertEqual(d[0], '127.0.0.1')
self.assertEqual(d[1], client_port)
self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED or d[2] == db.DB_REPMGR_DISCONNECTED)
d = self.dbenvClient.repmgr_site_list()
self.assertEqual(len(d), 1)
d = d.values()[0]
self.assertEqual(d[0], '127.0.0.1')
self.assertEqual(d[1], master_port)
self.assertTrue(d[2] == db.DB_REPMGR_CONNECTED or d[2] == db.DB_REPMGR_DISCONNECTED)
if db.version() >= (4, 6):
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR)
self.assertTrue('msgs_queued' in d)
self.dbMaster = db.DB(self.dbenvMaster)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.open('test', db.DB_HASH, db.DB_CREATE, 438, txn=txn)
txn.commit()
import time, os.path
timeout = time.time() + 10
while time.time() < timeout and not os.path.exists(os.path.join(self.homeDirClient, 'test')):
time.sleep(0.01)
self.dbClient = db.DB(self.dbenvClient)
while True:
txn = self.dbenvClient.txn_begin()
try:
self.dbClient.open('test', db.DB_HASH, flags=db.DB_RDONLY, mode=438, txn=txn)
except db.DBRepHandleDeadError:
txn.abort()
self.dbClient.close()
self.dbClient = db.DB(self.dbenvClient)
continue
txn.commit()
break
txn = self.dbenvMaster.txn_begin()
self.dbMaster.put('ABC', '123', txn=txn)
txn.commit()
import time
timeout = time.time() + 10
v = None
while time.time() < timeout and v is None:
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get('ABC', txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
if time.time() >= timeout and startup_timeout:
self.skipTest('replication test skipped due to random failure, see issue 3892')
self.assertTrue(time.time() < timeout)
self.assertEqual('123', v)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.delete('ABC', txn=txn)
txn.commit()
timeout = time.time() + 10
while time.time() < timeout and v is not None:
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get('ABC', txn=txn)
txn.commit()
if v is None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEqual(None, v)
return
class DBEnvClosedEarlyCrash(unittest.TestCase):
def setUp(self):
self.homeDir = get_new_environment_path()
self.filename = "test"
def tearDown(self):
test_support.rmtree(self.homeDir)
def test01_close_dbenv_before_db(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
d = db.DB(dbenv)
d2 = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
self.assertRaises(db.DBNoSuchFileError, d2.open, self.filename + "2",
db.DB_BTREE, db.DB_THREAD, 0666)
d.put("test", "this is a test")
self.assertEqual(d.get("test"), "this is a test", "put!=get")
dbenv.close() # This "close" should close the child db handle also
self.assertRaises(db.DBError, d.get, "test")
def test02_close_dbenv_before_dbcursor(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
d = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
d.put("test", "this is a test")
d.put("test2", "another test")
d.put("test3", "another one")
self.assertEqual(d.get("test"), "this is a test", "put!=get")
c = d.cursor()
c.first()
c.next()
d.close() # This "close" should close the child db handle also
# db.close should close the child cursor
self.assertRaises(db.DBError, c.next)
d = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
c = d.cursor()
c.first()
c.next()
dbenv.close()
# The "close" should close the child db handle also, with cursors
self.assertRaises(db.DBError, c.next)
def test03_close_db_before_dbcursor_without_env(self):
import os.path
path = os.path.join(self.homeDir, self.filename)
d = db.DB()
d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
d.put("test", "this is a test")
d.put("test2", "another test")
d.put("test3", "another one")
self.assertEqual(d.get("test"), "this is a test", "put!=get")
c = d.cursor()
c.first()
c.next()
d.close()
# The "close" should close the child db handle also
self.assertRaises(db.DBError, c.next)
def test04_close_massive(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
dbs = [db.DB(dbenv) for i in xrange(16)]
cursors = []
for i in dbs:
i.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD,
0666)
dbs[10].put("test", "this is a test")
dbs[10].put("test2", "another test")
dbs[10].put("test3", "another one")
self.assertEqual(dbs[4].get("test"), "this is a test", "put!=get")
for i in dbs:
cursors.extend([i.cursor() for j in xrange(32)])
for i in dbs[::3]:
i.close()
for i in cursors[::3]:
i.close()
# Check for missing exception in DB! (after DB close)
self.assertRaises(db.DBError, dbs[9].get, "test")
# Check for missing exception in DBCursor! (after DB close)
self.assertRaises(db.DBError, cursors[101].first)
cursors[80].first()
cursors[80].next()
dbenv.close() # This "close" should close the child db handle also
# Check for missing exception! (after DBEnv close)
self.assertRaises(db.DBError, cursors[80].next)
def test05_close_dbenv_delete_db_success(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_CDB | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_MPOOL,
0666)
d = db.DB(dbenv)
d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
dbenv.close() # This "close" should close the child db handle also
del d
try:
import gc
except ImportError:
gc = None
if gc:
# force d.__del__ [DB_dealloc] to be called
gc.collect()
def test06_close_txn_before_dup_cursor(self):
dbenv = db.DBEnv()
dbenv.open(
self.homeDir,
db.DB_INIT_TXN | db.DB_INIT_MPOOL | db.DB_INIT_LOG | db.DB_CREATE)
d = db.DB(dbenv)
txn = dbenv.txn_begin()
if db.version() < (4, 1):
d.open(self.filename, dbtype=db.DB_HASH, flags=db.DB_CREATE)
else:
d.open(self.filename,
dbtype=db.DB_HASH,
flags=db.DB_CREATE,
txn=txn)
d.put("XXX", "yyy", txn=txn)
txn.commit()
txn = dbenv.txn_begin()
c1 = d.cursor(txn)
c2 = c1.dup()
self.assertEquals(("XXX", "yyy"), c1.first())
import warnings
# Not interested in warnings about implicit close.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
txn.commit()
self.assertRaises(db.DBCursorClosedError, c2.first)
if db.version() > (4, 3, 0):
def test07_close_db_before_sequence(self):
import os.path
path = os.path.join(self.homeDir, self.filename)
d = db.DB()
d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666)
dbs = db.DBSequence(d)
d.close() # This "close" should close the child DBSequence also
dbs.close() # If not closed, core dump (in Berkeley DB 4.6.*)
def test_suite():
suite = unittest.TestSuite()
if db.version() >= (4,3):
suite.addTest(unittest.makeSuite(DBSequenceTest))
return suite
class DBSequenceTest(unittest.TestCase):
def setUp(self):
self.int_32_max = 4294967296L
self.homeDir = get_new_environment_path()
self.filename = 'test'
self.dbenv = db.DBEnv()
self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL, 438)
self.d = db.DB(self.dbenv)
self.d.open(self.filename, db.DB_BTREE, db.DB_CREATE, 438)
def tearDown(self):
if hasattr(self, 'seq'):
self.seq.close()
del self.seq
if hasattr(self, 'd'):
self.d.close()
del self.d
if hasattr(self, 'dbenv'):
self.dbenv.close()
del self.dbenv
test_support.rmtree(self.homeDir)
def test_get(self):
self.seq = db.DBSequence(self.d, flags=0)
start_value = 10 * self.int_32_max
self.assertEqual(42949672960L, start_value)
self.assertEqual(None, self.seq.initial_value(start_value))
self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE))
self.assertEqual(start_value, self.seq.get(5))
self.assertEqual(start_value + 5, self.seq.get())
return
def test_remove(self):
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(None, self.seq.remove(txn=None, flags=0))
del self.seq
return
def test_get_key(self):
self.seq = db.DBSequence(self.d, flags=0)
key = 'foo'
self.assertEqual(None, self.seq.open(key=key, txn=None, flags=db.DB_CREATE))
self.assertEqual(key, self.seq.get_key())
return
def test_get_dbp(self):
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(self.d, self.seq.get_dbp())
return
def test_cachesize(self):
self.seq = db.DBSequence(self.d, flags=0)
cashe_size = 10
self.assertEqual(None, self.seq.set_cachesize(cashe_size))
self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(cashe_size, self.seq.get_cachesize())
return
def test_flags(self):
self.seq = db.DBSequence(self.d, flags=0)
flag = db.DB_SEQ_WRAP
self.assertEqual(None, self.seq.set_flags(flag))
self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(flag, self.seq.get_flags() & flag)
return
def test_range(self):
self.seq = db.DBSequence(self.d, flags=0)
seq_range = (10 * self.int_32_max, 11 * self.int_32_max - 1)
self.assertEqual(None, self.seq.set_range(seq_range))
self.seq.initial_value(seq_range[0])
self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
self.assertEqual(seq_range, self.seq.get_range())
return
def test_stat(self):
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.open(key='foo', txn=None, flags=db.DB_CREATE))
stat = self.seq.stat()
for param in ('nowait', 'min', 'max', 'value', 'current', 'flags', 'cache_size', 'last_value', 'wait'):
self.assertTrue(param in stat, "parameter %s isn't in stat info" % param)
return
if db.version() >= (4, 7):
def test_stat_crash(self):
d = db.DB()
d.open(None, dbtype=db.DB_HASH, flags=db.DB_CREATE)
seq = db.DBSequence(d, flags=0)
self.assertRaises(db.DBNotFoundError, seq.open, key='id', txn=None, flags=0)
self.assertRaises(db.DBInvalidArgError, seq.stat)
d.close()
return
def test_64bits(self):
value_plus = 9223372036854775806L
self.assertEqual(9223372036854775806L, value_plus)
value_minus = -9223372036854775807L
self.assertEqual(-9223372036854775807L, value_minus)
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.initial_value(value_plus - 1))
self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE))
self.assertEqual(value_plus - 1, self.seq.get(1))
self.assertEqual(value_plus, self.seq.get(1))
self.seq.remove(txn=None, flags=0)
self.seq = db.DBSequence(self.d, flags=0)
self.assertEqual(None, self.seq.initial_value(value_minus))
self.assertEqual(None, self.seq.open(key='id', txn=None, flags=db.DB_CREATE))
self.assertEqual(value_minus, self.seq.get(1))
self.assertEqual(value_minus + 1, self.seq.get(1))
return
def test_multiple_close(self):
self.seq = db.DBSequence(self.d)
self.seq.close()
self.seq.close()
self.seq.close()
def _destroy_env(self):
if self.nosync or (db.version()[:2] == (4,6)): # Known bug
self.dbenv.log_flush()
self.db.close()
self.dbenv.close()
def test01_basic_replication(self):
master_port = test_support.find_unused_port()
self.dbenvMaster.repmgr_set_local_site("127.0.0.1", master_port)
client_port = test_support.find_unused_port()
self.dbenvClient.repmgr_set_local_site("127.0.0.1", client_port)
self.dbenvMaster.repmgr_add_remote_site("127.0.0.1", client_port)
self.dbenvClient.repmgr_add_remote_site("127.0.0.1", master_port)
self.dbenvMaster.rep_set_nsites(2)
self.dbenvClient.rep_set_nsites(2)
self.dbenvMaster.rep_set_priority(10)
self.dbenvClient.rep_set_priority(0)
self.dbenvMaster.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100123)
self.dbenvClient.rep_set_timeout(db.DB_REP_CONNECTION_RETRY, 100321)
self.assertEquals(
self.dbenvMaster.rep_get_timeout(db.DB_REP_CONNECTION_RETRY),
100123)
self.assertEquals(
self.dbenvClient.rep_get_timeout(db.DB_REP_CONNECTION_RETRY),
100321)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100234)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_TIMEOUT, 100432)
self.assertEquals(
self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT),
100234)
self.assertEquals(
self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_TIMEOUT),
100432)
self.dbenvMaster.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100345)
self.dbenvClient.rep_set_timeout(db.DB_REP_ELECTION_RETRY, 100543)
self.assertEquals(
self.dbenvMaster.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100345)
self.assertEquals(
self.dbenvClient.rep_get_timeout(db.DB_REP_ELECTION_RETRY), 100543)
self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL)
self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER)
self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT)
self.assertEquals(self.dbenvMaster.rep_get_nsites(), 2)
self.assertEquals(self.dbenvClient.rep_get_nsites(), 2)
self.assertEquals(self.dbenvMaster.rep_get_priority(), 10)
self.assertEquals(self.dbenvClient.rep_get_priority(), 0)
self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(),
db.DB_REPMGR_ACKS_ALL)
# The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE
# is not generated if the master has no new transactions.
# This is solved in BDB 4.6 (#15542).
import time
timeout = time.time() + 60
while (time.time() < timeout) and not (self.confirmed_master
and self.client_startupdone):
time.sleep(0.02)
# self.client_startupdone does not always get set to True within
# the timeout. On windows this may be a deep issue, on other
# platforms it is likely just a timing issue, especially on slow
# virthost buildbots (see issue 3892 for more). Even though
# the timeout triggers, the rest of this test method usually passes
# (but not all of it always, see below). So we just note the
# timeout on stderr and keep soldering on.
if time.time() > timeout:
import sys
print >> sys.stderr, ("XXX: timeout happened before"
"startup was confirmed - see issue 3892")
startup_timeout = True
d = self.dbenvMaster.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], client_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
d = self.dbenvClient.repmgr_site_list()
self.assertEquals(len(d), 1)
self.assertEquals(d[0][0], "127.0.0.1")
self.assertEquals(d[0][1], master_port)
self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \
(d[0][2]==db.DB_REPMGR_DISCONNECTED))
if db.version() >= (4, 6):
d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR)
self.assertTrue("msgs_queued" in d)
self.dbMaster = db.DB(self.dbenvMaster)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn)
txn.commit()
import time, os.path
timeout = time.time() + 10
while (time.time()<timeout) and \
not (os.path.exists(os.path.join(self.homeDirClient,"test"))) :
time.sleep(0.01)
self.dbClient = db.DB(self.dbenvClient)
while True:
txn = self.dbenvClient.txn_begin()
try:
self.dbClient.open("test",
db.DB_HASH,
flags=db.DB_RDONLY,
mode=0666,
txn=txn)
except db.DBRepHandleDeadError:
txn.abort()
self.dbClient.close()
self.dbClient = db.DB(self.dbenvClient)
continue
txn.commit()
break
txn = self.dbenvMaster.txn_begin()
self.dbMaster.put("ABC", "123", txn=txn)
txn.commit()
import time
timeout = time.time() + 10
v = None
while (time.time() < timeout) and (v == None):
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get("ABC", txn=txn)
txn.commit()
if v == None:
time.sleep(0.02)
# If startup did not happen before the timeout above, then this test
# sometimes fails. This happens randomly, which causes buildbot
# instability, but all the other bsddb tests pass. Since bsddb3 in the
# stdlib is currently not getting active maintenance, and is gone in
# py3k, we just skip the end of the test in that case.
if time.time() >= timeout and startup_timeout:
self.skipTest("replication test skipped due to random failure, "
"see issue 3892")
self.assertTrue(time.time() < timeout)
self.assertEquals("123", v)
txn = self.dbenvMaster.txn_begin()
self.dbMaster.delete("ABC", txn=txn)
txn.commit()
timeout = time.time() + 10
while (time.time() < timeout) and (v != None):
txn = self.dbenvClient.txn_begin()
v = self.dbClient.get("ABC", txn=txn)
txn.commit()
if v == None:
time.sleep(0.02)
self.assertTrue(time.time() < timeout)
self.assertEquals(None, v)
