def test06_Transactions(self): d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test06_Transactions..." % self.__class__.__name__ assert d.get('new rec', txn=self.txn) == None d.put('new rec', 'this is a new record', self.txn) assert d.get('new rec', txn=self.txn) == 'this is a new record' self.txn.abort() assert d.get('new rec') == None self.txn = self.env.txn_begin() assert d.get('new rec', txn=self.txn) == None d.put('new rec', 'this is a new record', self.txn) assert d.get('new rec', txn=self.txn) == 'this is a new record' self.txn.commit() assert d.get('new rec') == 'this is a new record' self.txn = self.env.txn_begin() c = d.cursor(self.txn) rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec rec = c.next() assert count == self._numKeys + 1 c.close() # Cursors *MUST* be closed before commit! self.txn.commit() # flush pending updates try: self.env.txn_checkpoint(0, 0, 0) except db.DBIncompleteError: pass if db.version() >= (4, 0): statDict = self.env.log_stat(0) assert statDict.has_key('magic') assert statDict.has_key('version') assert statDict.has_key('cur_file') assert statDict.has_key('region_nowait') # must have at least one log file present: logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG) assert logs != None for log in logs: if verbose: print 'log file: ' + log if db.version() >= (4, 2): logs = self.env.log_archive(db.DB_ARCH_REMOVE) assert not logs self.txn = self.env.txn_begin()
def test06_Transactions(self): d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test06_Transactions..." % self.__class__.__name__ assert d.get('new rec', txn=self.txn) == None d.put('new rec', 'this is a new record', self.txn) assert d.get('new rec', txn=self.txn) == 'this is a new record' self.txn.abort() assert d.get('new rec') == None self.txn = self.env.txn_begin() assert d.get('new rec', txn=self.txn) == None d.put('new rec', 'this is a new record', self.txn) assert d.get('new rec', txn=self.txn) == 'this is a new record' self.txn.commit() assert d.get('new rec') == 'this is a new record' self.txn = self.env.txn_begin() c = d.cursor(self.txn) rec = c.first() count = 0 while rec is not None: count = count + 1 if verbose and count % 100 == 0: print rec rec = c.next() assert count == self._numKeys+1 c.close() # Cursors *MUST* be closed before commit! self.txn.commit() # flush pending updates try: self.env.txn_checkpoint (0, 0, 0) except db.DBIncompleteError: pass if db.version() >= (4,0): statDict = self.env.log_stat(0); assert statDict.has_key('magic') assert statDict.has_key('version') assert statDict.has_key('cur_file') assert statDict.has_key('region_nowait') # must have at least one log file present: logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG) assert logs != None for log in logs: if verbose: print 'log file: ' + log if db.version() >= (4,2): logs = self.env.log_archive(db.DB_ARCH_REMOVE) assert not logs self.txn = self.env.txn_begin()
def print_versions(): print print '-=' * 38 print db.DB_VERSION_STRING print 'bsddb.db.version(): %s' % (db.version(),) if db.version() >= (5, 0): print 'bsddb.db.full_version(): %s' % repr(db.full_version()) print 'bsddb.db.__version__: %s' % db.__version__ print 'bsddb.db.cvsid: %s' % db.cvsid suffix = '__' print 'py module: %s' % getattr(bsddb, '__file' + suffix) print 'extension module: %s' % getattr(bsddb, '__file' + suffix) print 'python version: %s' % sys.version print 'My pid: %s' % os.getpid() print '-=' * 38
def print_versions(): print print '-=' * 38 print db.DB_VERSION_STRING print 'bsddb.db.version(): %s' % (db.version(), ) if db.version() >= (5, 0): print 'bsddb.db.full_version(): %s' % repr(db.full_version()) print 'bsddb.db.__version__: %s' % db.__version__ print 'bsddb.db.cvsid: %s' % db.cvsid suffix = '__' print 'py module: %s' % getattr(bsddb, '__file' + suffix) print 'extension module: %s' % getattr(bsddb, '__file' + suffix) print 'python version: %s' % sys.version print 'My pid: %s' % os.getpid() print '-=' * 38
def get_dbdir_summary(self, dirpath, name): """ Returns (people_count, bsddb_version, schema_version) of current DB. Returns ("Unknown", "Unknown", "Unknown") if invalid DB or other error. """ if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3: from bsddb3 import dbshelve, db else: from bsddb import dbshelve, db from gramps.gen.db import META, PERSON_TBL from gramps.gen.db.dbconst import BDBVERSFN bdbversion_file = os.path.join(dirpath, BDBVERSFN) if os.path.isfile(bdbversion_file): vers_file = open(bdbversion_file) bsddb_version = vers_file.readline().strip() else: return "Unknown", "Unknown", "Unknown" current_bsddb_version = str(db.version()) if bsddb_version != current_bsddb_version: return "Unknown", bsddb_version, "Unknown" env = db.DBEnv() flags = db.DB_CREATE | db.DB_PRIVATE |\ db.DB_INIT_MPOOL |\ db.DB_INIT_LOG | db.DB_INIT_TXN try: env.open(dirpath, flags) except Exception as msg: LOG.warning("Error opening db environment for '%s': %s" % (name, str(msg))) try: env.close() except Exception as msg: LOG.warning("Error closing db environment for '%s': %s" % (name, str(msg))) return "Unknown", bsddb_version, "Unknown" dbmap1 = dbshelve.DBShelf(env) fname = os.path.join(dirpath, META + ".db") try: dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY) except: env.close() return "Unknown", bsddb_version, "Unknown" schema_version = dbmap1.get(b'version', default=None) dbmap1.close() dbmap2 = dbshelve.DBShelf(env) fname = os.path.join(dirpath, PERSON_TBL + ".db") try: dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY) except: env.close() return "Unknown", bsddb_version, schema_version count = len(dbmap2) dbmap2.close() env.close() return (count, bsddb_version, schema_version)
def __init__(self, path, name, duplicates=False): self.path = path self.dbpath = os.path.join(self.path, name) if not os.path.isdir(self.path): os.makedirs(self.path) self.env = bdb.DBEnv() self.env.set_tx_max(self.max_txns) self.env.set_lk_max_lockers(self.max_txns*2) self.env.set_lk_max_locks(self.max_txns*2) self.env.set_lk_max_objects(self.max_txns*2) self.env.set_lk_detect(bdb.DB_LOCK_DEFAULT) self.env.set_flags(bdb.DB_TXN_NOSYNC, True) if hasattr(self.env, "log_set_config"): self.env.log_set_config(bdb.DB_LOG_AUTO_REMOVE, True) self.env.open(self.path, bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG | bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN | bdb.DB_RECOVER | bdb.DB_THREAD) self.db = bdb.DB(self.env) if duplicates: self.db.set_flags(bdb.DB_DUPSORT) if bdb.version() > (4,1): txn = self.env.txn_begin() self.db.open(self.dbpath, name, flags=bdb.DB_CREATE|bdb.DB_THREAD, dbtype=bdb.DB_BTREE, txn=txn) txn.commit() else: self.db.open(self.dbpath, name, flags=bdb.DB_CREATE|bdb.DB_THREAD, dbtype=bdb.DB_BTREE) self.thread = DatabaseManagerThread(self.env) self.thread.start()
def test13_associate_in_transaction(self): if verbose: print '\n', '-=' * 30 print "Running %s.test13_associateAutoCommit..." % \ self.__class__.__name__ txn = self.env.txn_begin() try: self.createDB(txn=txn) self.secDB = db.DB(self.env) self.secDB.set_flags(db.DB_DUP) self.secDB.set_get_returns_none(2) self.secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, txn=txn) if db.version() >= (4,1): self.getDB().associate(self.secDB, self.getGenre, txn=txn) else: self.getDB().associate(self.secDB, self.getGenre) self.addDataToDB(self.getDB(), txn=txn) except: txn.abort() raise self.txn_finish_test(self.secDB, txn=txn)
def test13_associate_in_transaction(self): if verbose: print '\n', '-=' * 30 print "Running %s.test13_associateAutoCommit..." % \ self.__class__.__name__ txn = self.env.txn_begin() try: self.createDB(txn=txn) self.secDB = db.DB(self.env) self.secDB.set_flags(db.DB_DUP) self.secDB.set_get_returns_none(2) self.secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, txn=txn) if db.version() >= (4, 1): self.getDB().associate(self.secDB, self.getGenre, txn=txn) else: self.getDB().associate(self.secDB, self.getGenre) self.addDataToDB(self.getDB(), txn=txn) except: txn.abort() raise self.txn_finish_test(self.secDB, txn=txn)
class BasicWithEnvTestCase(BasicTestCase): dbopenflags = db.DB_THREAD useEnv = 1 envflags = db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK #---------------------------------------- def test07_EnvRemoveAndRename(self): if not self.env: return if verbose: print '\n', '-=' * 30 print "Running %s.test07_EnvRemoveAndRename..." % self.__class__.__name__ # can't rename or remove an open DB self.d.close() newname = self.filename + '.renamed' self.env.dbrename(self.filename, None, newname) self.env.dbremove(newname) # dbremove and dbrename are in 4.1 and later if db.version() < (4, 1): del test07_EnvRemoveAndRename
def test_DB_set_flags_persists(self): if db.version() < (4,2): # The get_flags API required for this to work is only available # in BerkeleyDB >= 4.2 return try: db1 = db.DB() db1.set_flags(db.DB_DUPSORT) db1.open(self.filename, db.DB_HASH, db.DB_CREATE) db1['a'] = 'eh' db1['a'] = 'A' self.assertEqual([('a', 'A')], db1.items()) db1.put('a', 'Aa') self.assertEqual([('a', 'A'), ('a', 'Aa')], db1.items()) db1.close() db1 = db.DB() # no set_flags call, we're testing that it reads and obeys # the flags on open. db1.open(self.filename, db.DB_HASH) self.assertEqual([('a', 'A'), ('a', 'Aa')], db1.items()) # if it read the flags right this will replace all values # for key 'a' instead of adding a new one. (as a dict should) db1['a'] = 'new A' self.assertEqual([('a', 'new A')], db1.items()) finally: db1.close() os.unlink(self.filename)
def test_suite(): res = unittest.TestSuite() res.addTest(unittest.makeSuite(ComparatorTests)) if db.version() >= (3, 3, 11): res.addTest(unittest.makeSuite(BtreeExceptionsTestCase)) res.addTest(unittest.makeSuite(BtreeKeyCompareTestCase)) return res
def test_suite (): res = unittest.TestSuite () res.addTest (unittest.makeSuite (ComparatorTests)) if db.version () >= (3, 3, 11): res.addTest (unittest.makeSuite (BtreeExceptionsTestCase)) res.addTest (unittest.makeSuite (BtreeKeyCompareTestCase)) return res
def test00_version(self): info = db.version() if verbose: print "\n", "-=" * 20 print "bsddb.db.version(): %s" % (info,) print db.DB_VERSION_STRING print "-=" * 20 assert info == (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, db.DB_VERSION_PATCH)
def print_versions(): print() print('-=' * 38) print(db.DB_VERSION_STRING) print('bsddb.db.version(): %s' % (db.version(), )) if db.version() >= (5, 0) : print('bsddb.db.full_version(): %s' %repr(db.full_version())) print('bsddb.db.__version__: %s' % db.__version__) print('bsddb.db.cvsid: %s' % db.cvsid) # Workaround for allowing generating an EGGs as a ZIP files. suffix="__" print('py module: %s' % getattr(bsddb, "__file"+suffix)) print('extension module: %s' % getattr(bsddb, "__file"+suffix)) print('python version: %s' % sys.version) print('My pid: %s' % os.getpid()) print('-=' * 38)
def get_dbdir_summary(self, dirpath, name): """ Returns (people_count, bsddb_version, schema_version) of current DB. Returns ("Unknown", "Unknown", "Unknown") if invalid DB or other error. """ from bsddb3 import dbshelve, db from gramps.gen.db import META, PERSON_TBL from gramps.gen.db.dbconst import BDBVERSFN bdbversion_file = os.path.join(dirpath, BDBVERSFN) if os.path.isfile(bdbversion_file): vers_file = open(bdbversion_file) bsddb_version = vers_file.readline().strip() else: return "Unknown", "Unknown", "Unknown" current_bsddb_version = str(db.version()) if bsddb_version != current_bsddb_version: return "Unknown", bsddb_version, "Unknown" env = db.DBEnv() flags = db.DB_CREATE | db.DB_PRIVATE |\ db.DB_INIT_MPOOL |\ db.DB_INIT_LOG | db.DB_INIT_TXN try: env.open(dirpath, flags) except Exception as msg: LOG.warning("Error opening db environment for '%s': %s" % (name, str(msg))) try: env.close() except Exception as msg: LOG.warning("Error closing db environment for '%s': %s" % (name, str(msg))) return "Unknown", bsddb_version, "Unknown" dbmap1 = dbshelve.DBShelf(env) fname = os.path.join(dirpath, META + ".db") try: dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY) except: env.close() return "Unknown", bsddb_version, "Unknown" schema_version = dbmap1.get(b'version', default=None) dbmap1.close() dbmap2 = dbshelve.DBShelf(env) fname = os.path.join(dirpath, PERSON_TBL + ".db") try: dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY) except: env.close() return "Unknown", bsddb_version, schema_version count = len(dbmap2) dbmap2.close() env.close() return (count, bsddb_version, schema_version)
def test00_version(self): info = db.version() if verbose: print '\n', '-=' * 20 print 'bsddb.db.version(): %s' % (info, ) print db.DB_VERSION_STRING print '-=' * 20 assert info == (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, db.DB_VERSION_PATCH)
def print_versions(): print print '-=' * 38 print db.DB_VERSION_STRING print 'bsddb.db.version(): %s' % (db.version(), ) if db.version() >= (5, 0): print 'bsddb.db.full_version(): %s' % repr(db.full_version()) print 'bsddb.db.__version__: %s' % db.__version__ print 'bsddb.db.cvsid: %s' % db.cvsid # Workaround for allowing generating an EGGs as a ZIP files. suffix = "__" print 'py module: %s' % getattr(bsddb, "__file" + suffix) print 'extension module: %s' % getattr(bsddb, "__file" + suffix) print 'python version: %s' % sys.version print 'My pid: %s' % os.getpid() print '-=' * 38
def print_versions(): print print '-=' * 38 print db.DB_VERSION_STRING print 'bsddb.db.version(): %s' % (db.version(), ) print 'bsddb.db.__version__: %s' % db.__version__ print 'bsddb.db.cvsid: %s' % db.cvsid print 'python version: %s' % sys.version print 'My pid: %s' % os.getpid() print '-=' * 38
def print_versions(): print print "-=" * 38 print db.DB_VERSION_STRING print "bsddb.db.version(): %s" % (db.version(),) print "bsddb.db.__version__: %s" % db.__version__ print "bsddb.db.cvsid: %s" % db.cvsid print "python version: %s" % sys.version print "My pid: %s" % os.getpid() print "-=" * 38
def createDB(self, txn=None): self.cur = None self.secDB = None self.primary = db.DB(self.env) self.primary.set_get_returns_none(2) if db.version() >= (4, 1): self.primary.open(self.filename, "primary", self.dbtype, db.DB_CREATE | db.DB_THREAD | self.dbFlags, txn=txn) else: self.primary.open(self.filename, "primary", self.dbtype, db.DB_CREATE | db.DB_THREAD | self.dbFlags)
def print_versions(): print print '-=' * 38 print db.DB_VERSION_STRING print 'bsddb.db.version(): %s' % (db.version(), ) if db.version() >= (5, 0) : print 'bsddb.db.full_version(): %s' %repr(db.full_version()) print 'bsddb.db.__version__: %s' % db.__version__ # Workaround for allowing generating an EGGs as a ZIP files. suffix="__" print 'py module: %s' % getattr(bsddb, "__file"+suffix) print 'extension module: %s' % getattr(bsddb, "__file"+suffix) print 'Test working dir: %s' % get_test_path_prefix() import platform print 'python version: %s %s' % \ (sys.version.replace("\r", "").replace("\n", ""), \ platform.architecture()[0]) print 'My pid: %s' % os.getpid() print '-=' * 38
def print_versions(): print() print('-=' * 38) print(db.DB_VERSION_STRING) print('bsddb.db.version(): %s' % (db.version(), )) print('bsddb.db.__version__: %s' % db.__version__) print('bsddb.db.cvsid: %s' % db.cvsid) print('py module: %s' % bsddb.__file__) print('extension module: %s' % bsddb._bsddb.__file__) print('python version: %s' % sys.version) print('My pid: %s' % os.getpid()) print('-=' * 38)
def print_versions(): print() print('-=' * 38) print(db.DB_VERSION_STRING) print('bsddb.db.version(): %s' % (db.version(), )) if db.version() >= (5, 0) : print('bsddb.db.full_version(): %s' %repr(db.full_version())) print('bsddb.db.__version__: %s' % db.__version__) # Workaround for allowing generating an EGGs as a ZIP files. suffix="__" print('py module: %s' % getattr(bsddb, "__file"+suffix)) print('extension module: %s' % getattr(bsddb, "__file"+suffix)) print('Test working dir: %s' % get_test_path_prefix()) import platform print('python version: %s %s' % \ (sys.version.replace("\r", "").replace("\n", ""), \ platform.architecture()[0])) print('My pid: %s' % os.getpid()) print('-=' * 38)
def __init__(self): if db.version() < (4,1): return self.env = db.DBEnv() self.env.open(env_name, db.DB_CREATE | db.DB_INIT_TXN | db.DB_INIT_MPOOL) self.the_txn = self.env.txn_begin() self.map = db.DB(self.env) self.map.open('xxx.db', "p", db.DB_HASH, db.DB_CREATE, 0666, txn=self.the_txn) del self.env del self.the_txn
def test_suite(): suite = unittest.TestSuite() if db.version() >= (3, 3, 11): suite.addTest(unittest.makeSuite(AssociateErrorTestCase)) suite.addTest(unittest.makeSuite(AssociateHashTestCase)) suite.addTest(unittest.makeSuite(AssociateBTreeTestCase)) suite.addTest(unittest.makeSuite(AssociateRecnoTestCase)) if db.version() >= (4, 1): suite.addTest(unittest.makeSuite(AssociateBTreeTxnTestCase)) suite.addTest(unittest.makeSuite(ShelveAssociateHashTestCase)) suite.addTest(unittest.makeSuite(ShelveAssociateBTreeTestCase)) suite.addTest(unittest.makeSuite(ShelveAssociateRecnoTestCase)) if have_threads: suite.addTest(unittest.makeSuite(ThreadedAssociateHashTestCase)) suite.addTest(unittest.makeSuite(ThreadedAssociateBTreeTestCase)) suite.addTest(unittest.makeSuite(ThreadedAssociateRecnoTestCase)) return suite
def test06_Truncate(self): if db.version() < (3, 3): # truncate is a feature of BerkeleyDB 3.3 and above return d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test99_Truncate..." % self.__class__.__name__ d.put("abcde", "ABCDE") num = d.truncate() assert num >= 1, "truncate returned <= 0 on non-empty database" num = d.truncate() assert num == 0, "truncate on empty DB returned nonzero (%r)" % (num, )
def test06_Truncate(self): if db.version() < (3,3): # truncate is a feature of BerkeleyDB 3.3 and above return d = self.d if verbose: print '\n', '-=' * 30 print "Running %s.test99_Truncate..." % self.__class__.__name__ d.put("abcde", "ABCDE"); num = d.truncate() assert num >= 1, "truncate returned <= 0 on non-empty database" num = d.truncate() assert num == 0, "truncate on empty DB returned nonzero (%r)" % (num,)
def print_versions(): try: # For Pythons w/distutils pybsddb from bsddb3 import db except ImportError: # For Python 2.3 from bsddb import db print print '-=' * 38 print db.DB_VERSION_STRING print 'bsddb.db.version(): %s' % (db.version(), ) print 'bsddb.db.__version__: %s' % db.__version__ print 'bsddb.db.cvsid: %s' % db.cvsid print 'python version: %s' % sys.version print 'My pid: %s' % os.getpid() print '-=' * 38
def print_versions(): print print "-=" * 38 print db.DB_VERSION_STRING print "bsddb.db.version(): %s" % (db.version(),) print "bsddb.db.__version__: %s" % db.__version__ print "bsddb.db.cvsid: %s" % db.cvsid # Workaround for allowing generating an EGGs as a ZIP files. suffix = "__" print "py module: %s" % getattr(bsddb, "__file" + suffix) print "extension module: %s" % getattr(bsddb, "__file" + suffix) print "python version: %s" % sys.version print "My pid: %s" % os.getpid() print "-=" * 38
def test01_simple(self): if verbose: print '\n', '-=' * 30 print "Running %s.test01_simple..." % self.__class__.__name__ anID = self.env.lock_id() if verbose: print "locker ID: %s" % anID lock = self.env.lock_get(anID, "some locked thing", db.DB_LOCK_WRITE) if verbose: print "Aquired lock: %s" % lock time.sleep(1) self.env.lock_put(lock) if verbose: print "Released lock: %s" % lock if db.version() >= (4,0): self.env.lock_id_free(anID)
def test07_TxnTruncate(self): if db.version() < (3, 3): # truncate is a feature of BerkeleyDB 3.3 and above return d = self.d if verbose: print "\n", "-=" * 30 print "Running %s.test07_TxnTruncate..." % self.__class__.__name__ d.put("abcde", "ABCDE") txn = self.env.txn_begin() num = d.truncate(txn) assert num >= 1, "truncate returned <= 0 on non-empty database" num = d.truncate(txn) assert num == 0, "truncate on empty DB returned nonzero (%r)" % (num,) txn.commit()
def theThread(self, sleepTime, lockType): name = currentThread().getName() if lockType == db.DB_LOCK_WRITE: lt = "write" else: lt = "read" anID = self.env.lock_id() if verbose: print "%s: locker ID: %s" % (name, anID) lock = self.env.lock_get(anID, "some locked thing", lockType) if verbose: print "%s: Aquired %s lock: %s" % (name, lt, lock) time.sleep(sleepTime) self.env.lock_put(lock) if verbose: print "%s: Released %s lock: %s" % (name, lt, lock) if db.version() >= (4,0): self.env.lock_id_free(anID)
def test_suite(): suite = unittest.TestSuite() if db.version() >= (4, 3): suite.addTest(unittest.makeSuite(DBSequenceTest)) return suite
def test02_basicPost32(self): # Basic Queue tests using the new DB.consume method in DB 3.2+ # (No cursor needed) if verbose: print '\n', '-=' * 30 print "Running %s.test02_basicPost32..." % self.__class__.__name__ if db.version() < (3, 2, 0): if verbose: print "Test not run, DB not new enough..." return d = db.DB() d.set_re_len(40) # Queues must be fixed length d.open(self.filename, db.DB_QUEUE, db.DB_CREATE) if verbose: print "before appends" + '-' * 30 pprint(d.stat()) for x in string.letters: d.append(x * 40) assert len(d) == 52 d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") assert len(d) == 55 if verbose: print "before close" + '-' * 30 pprint(d.stat()) d.close() del d d = db.DB() d.open(self.filename) #d.set_get_returns_none(true) if verbose: print "after open" + '-' * 30 pprint(d.stat()) d.append("one more") if verbose: print "after append" + '-' * 30 pprint(d.stat()) rec = d.consume() while rec: if verbose: print rec rec = d.consume() if verbose: print "after consume loop" + '-' * 30 pprint(d.stat()) d.close()
def test_suite(): suite = unittest.TestSuite() if db.version() >= (4,3): suite.addTest(unittest.makeSuite(DBSequenceTest)) return suite
def __init__(self, **kwargs): # create db environment additional_flags = kwargs.get('flags', 0) recovery_mode = kwargs.get('recover', 0) if recovery_mode == 2: additional_flags |= db.DB_RECOVER_FATAL elif recovery_mode == 1: additional_flags |= db.DB_RECOVER if hasattr(db, 'DB_REGISTER'): additional_flags |= db.DB_REGISTER self._env = db.DBEnv() # ability to override settings' dir for testing purposes data_dir = kwargs.get('dir', self.dir) self._env.set_data_dir(data_dir) # ability to override settings' log_dir for testing purposes log_dir = kwargs.get('log_dir', kwargs.get('dir', self.log_dir)) self._env.set_lg_dir(log_dir) self._env.set_tx_max(self.txn_max) if self.txn_timeout is not None: if self.txn_timeout > 0: self._env.set_timeout(self.txn_timeout, db.DB_SET_TXN_TIMEOUT) else: self._env.set_flags(db.DB_TXN_NOWAIT, 1) if self.cache_size is not None: self._env.set_cachesize(*self.cache_size) if os.name != 'nt' and self.shm_key: self._env.set_shm_key(self.shm_key) additional_flags |= db.DB_SYSTEM_MEM # replication settings rep_config = settings['store'].get('rep_config', None) init_rep = kwargs.get('init_rep', False) if rep_config: # in replicated environments use non-durable transactions self._env.set_flags(db.DB_TXN_NOSYNC, 1) additional_flags |= db.DB_INIT_REP self._env.open(self.env_dir, db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_LOG | db.DB_INIT_TXN | db.DB_CREATE | additional_flags) db_flags = db.DB_THREAD | db.DB_AUTO_COMMIT | db.DB_CREATE | \ db.DB_MULTIVERSION db_mode = 0o660 if rep_config: from porcupine.db.bsddb.replication import ReplicationService # initialiaze replication service self.replication_service = \ ReplicationService(self._env, rep_config) if init_rep: # check multiprocessing is_multiprocess = services['main'].is_multiprocess or \ services['management'].is_multiprocess if is_multiprocess and int(rep_config['priority']) > 0 \ and db.version() < (4, 8): self._env.close() self.__remove_env() raise exceptions.ConfigurationError( 'Multiprocessing master candidates ' + 'require BerkeleyDB 4.8 or higher') # start replication service self.replication_service.start() # wait for client start-up timeout = time.time() + 20 while time.time() < timeout and \ not self.replication_service.is_master() and \ not self.replication_service.client_startup_done: time.sleep(0.02) timeout = time.time() + 20 while time.time() < timeout and \ not (os.path.exists( os.path.join(self.dir, 'porcupine.db'))): time.sleep(0.02) else: self.replication_service = None # open items db while True: self._itemdb = db.DB(self._env) self._itemdb.set_pagesize(2048) try: self._itemdb.open('porcupine.db', 'items', dbtype=db.DB_BTREE, mode=db_mode, flags=db_flags) except db.DBLockDeadlockError: self._itemdb.close() continue break # open documents db while True: self._docdb = db.DB(self._env) try: self._docdb.open('porcupine.db', 'docs', dbtype=db.DB_HASH, mode=db_mode, flags=db_flags) except db.DBLockDeadlockError: self._docdb.close() continue break # open indices self._indices = {} for name, unique, immutable in settings['store']['indices']: self._indices[name] = DbIndex(self._env, self._itemdb, name, unique, immutable, db_flags) self._running = True maintain = kwargs.get('maintain', False) if maintain and self._maintenance_thread is None: # start deadlock detector self._maintenance_thread = Thread(target=self.__maintain, name='DB maintenance thread') self._maintenance_thread.start() # start checkpoint thread self._checkpoint_thread = Thread(target=self.__checkpoint, name='DB checkpoint thread') self._checkpoint_thread.start()
import os from os import path as os_path, mkdir import threading from threading import Lock, current_thread, Event import errno from errno import EEXIST import pickle from pickle import dumps as pickle, loads as unpickle try: import bsddb3 as bsddb3 from bsddb3 import db as bsddb except ImportError: _no_bssdb3 = True else: db_major, db_minor = bsddb.version()[:2] _no_bssdb3 = False if __name__ == "__main__": # add pythomnic/lib to sys.path import os import sys main_module_dir = os.path.dirname( sys.modules["__main__"].__file__) or os.getcwd() sys.path.insert( 0, os.path.normpath(os.path.join(main_module_dir, "..", "..", "lib"))) import exc_string from exc_string import exc_string import typecheck from typecheck import typecheck, callable, optional import pmnc.perf_info
# Written by Ross Cohen # See LICENSE.txt for license information. from os import path try: from bsddb3 import db version_info = db.__version__.split('.') if version_info < [4,1]: raise ImportError if db.version() < (4,1): raise ImportError except ImportError: from bsddb import db version_info = db.__version__.split('.') if version_info < [4,1]: raise ImportError, 'bsddb 4.1 or higher is required' if db.version() < (4,1): raise ImportError, 'berkeleydb 4.1 or higher is required' history_format_version = 1 rebuild_format_version = 4 class VersionMismatchException(Exception): pass def check_format_version(dir): try: fd = open(path.join(dir, 'format'), 'r') ver = int(fd.read()) fd.close() except IOError:
def __init__(self, **kwargs): # create db environment additional_flags = kwargs.get('flags', 0) recovery_mode = kwargs.get('recover', 0) if recovery_mode == 2: additional_flags |= db.DB_RECOVER_FATAL elif recovery_mode == 1: additional_flags |= db.DB_RECOVER if hasattr(db, 'DB_REGISTER'): additional_flags |= db.DB_REGISTER self._env = db.DBEnv() # ability to override settings' dir for testing purposes data_dir = kwargs.get('dir', self.dir) self._env.set_data_dir(data_dir) # ability to override settings' log_dir for testing purposes log_dir = kwargs.get('log_dir', kwargs.get('dir', self.log_dir)) self._env.set_lg_dir(log_dir) self._env.set_tx_max(self.txn_max) if self.txn_timeout is not None: if self.txn_timeout > 0: self._env.set_timeout(self.txn_timeout, db.DB_SET_TXN_TIMEOUT) else: self._env.set_flags(db.DB_TXN_NOWAIT, 1) if self.cache_size is not None: self._env.set_cachesize(*self.cache_size) if os.name != 'nt' and self.shm_key: self._env.set_shm_key(self.shm_key) additional_flags |= db.DB_SYSTEM_MEM # replication settings rep_config = settings['store'].get('rep_config', None) init_rep = kwargs.get('init_rep', False) if rep_config: # in replicated environments use non-durable transactions self._env.set_flags(db.DB_TXN_NOSYNC, 1) additional_flags |= db.DB_INIT_REP self._env.open( self.env_dir, db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_LOG | db.DB_INIT_TXN | db.DB_CREATE | additional_flags) db_flags = db.DB_THREAD | db.DB_AUTO_COMMIT | db.DB_CREATE | \ db.DB_MULTIVERSION db_mode = 0o660 if rep_config: from porcupine.db.bsddb.replication import ReplicationService # initialiaze replication service self.replication_service = \ ReplicationService(self._env, rep_config) if init_rep: # check multiprocessing is_multiprocess = services['main'].is_multiprocess or \ services['management'].is_multiprocess if is_multiprocess and int(rep_config['priority']) > 0 \ and db.version() < (4, 8): self._env.close() self.__remove_env() raise exceptions.ConfigurationError( 'Multiprocessing master candidates ' + 'require BerkeleyDB 4.8 or higher') # start replication service self.replication_service.start() # wait for client start-up timeout = time.time() + 20 while time.time() < timeout and \ not self.replication_service.is_master() and \ not self.replication_service.client_startup_done: time.sleep(0.02) timeout = time.time() + 20 while time.time() < timeout and \ not (os.path.exists( os.path.join(self.dir, 'porcupine.db'))): time.sleep(0.02) else: self.replication_service = None # open items db while True: self._itemdb = db.DB(self._env) self._itemdb.set_pagesize(2048) try: self._itemdb.open('porcupine.db', 'items', dbtype=db.DB_BTREE, mode=db_mode, flags=db_flags) except db.DBLockDeadlockError: self._itemdb.close() continue break # open documents db while True: self._docdb = db.DB(self._env) try: self._docdb.open('porcupine.db', 'docs', dbtype=db.DB_HASH, mode=db_mode, flags=db_flags) except db.DBLockDeadlockError: self._docdb.close() continue break # open indices self._indices = {} for name, unique, immutable in settings['store']['indices']: self._indices[name] = DbIndex(self._env, self._itemdb, name, unique, immutable, db_flags) self._running = True maintain = kwargs.get('maintain', False) if maintain and self._maintenance_thread is None: # start deadlock detector self._maintenance_thread = Thread(target=self.__maintain, name='DB maintenance thread') self._maintenance_thread.start() # start checkpoint thread self._checkpoint_thread = Thread(target=self.__checkpoint, name='DB checkpoint thread') self._checkpoint_thread.start()
class DB(object): "Berkeley DB database interface" # data dir dir = os.path.abspath(settings['store']['bdb_data_dir']) if dir[-1] != '/': dir += '/' # log_dir log_dir = os.path.abspath(settings['store'].get('bdb_log_dir', dir)) if log_dir[-1] != '/': log_dir += '/' # environment files directory env_dir = os.path.abspath(settings['store'].get( 'env_dir', os.path.abspath(settings['global']['temp_folder']))) if env_dir[-1] != '/': env_dir += '/' # cache size cache_size = settings['store'].get('cache_size', None) # maximum concurrent transactions # due to snapshot isolation this should be kept high enough txn_max = settings['store'].get('max_tx', 1000) # transaction timeout txn_timeout = settings['store'].get('tx_timeout', None) # shared memory key shm_key = settings['store'].get('shm_key', None) # maintenance (deadlock detector) thread _maintenance_thread = None # checkpoint thread _checkpoint_thread = None # trickle thread _trickle_thread = None # log berkeleyDB version logger.info('BerkeleyDB version is %s' % '.'.join(str(x) for x in db.version())) def __init__(self, **kwargs): # create db environment additional_flags = kwargs.get('flags', 0) recovery_mode = kwargs.get('recover', 0) if recovery_mode == 2: additional_flags |= db.DB_RECOVER_FATAL elif recovery_mode == 1: additional_flags |= db.DB_RECOVER if hasattr(db, 'DB_REGISTER'): additional_flags |= db.DB_REGISTER self._env = db.DBEnv() # ability to override settings' dir for testing purposes data_dir = kwargs.get('dir', self.dir) self._env.set_data_dir(data_dir) # ability to override settings' log_dir for testing purposes log_dir = kwargs.get('log_dir', kwargs.get('dir', self.log_dir)) self._env.set_lg_dir(log_dir) self._env.set_tx_max(self.txn_max) if self.txn_timeout is not None: if self.txn_timeout > 0: self._env.set_timeout(self.txn_timeout, db.DB_SET_TXN_TIMEOUT) else: self._env.set_flags(db.DB_TXN_NOWAIT, 1) if self.cache_size is not None: self._env.set_cachesize(*self.cache_size) if os.name != 'nt' and self.shm_key: self._env.set_shm_key(self.shm_key) additional_flags |= db.DB_SYSTEM_MEM # replication settings rep_config = settings['store'].get('rep_config', None) init_rep = kwargs.get('init_rep', False) if rep_config: # in replicated environments use non-durable transactions self._env.set_flags(db.DB_TXN_NOSYNC, 1) additional_flags |= db.DB_INIT_REP self._env.open( self.env_dir, db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_LOG | db.DB_INIT_TXN | db.DB_CREATE | additional_flags) db_flags = db.DB_THREAD | db.DB_AUTO_COMMIT | db.DB_CREATE | \ db.DB_MULTIVERSION db_mode = 0o660 if rep_config: from porcupine.db.bsddb.replication import ReplicationService # initialiaze replication service self.replication_service = \ ReplicationService(self._env, rep_config) if init_rep: # check multiprocessing is_multiprocess = services['main'].is_multiprocess or \ services['management'].is_multiprocess if is_multiprocess and int(rep_config['priority']) > 0 \ and db.version() < (4, 8): self._env.close() self.__remove_env() raise exceptions.ConfigurationError( 'Multiprocessing master candidates ' + 'require BerkeleyDB 4.8 or higher') # start replication service self.replication_service.start() # wait for client start-up timeout = time.time() + 20 while time.time() < timeout and \ not self.replication_service.is_master() and \ not self.replication_service.client_startup_done: time.sleep(0.02) timeout = time.time() + 20 while time.time() < timeout and \ not (os.path.exists( os.path.join(self.dir, 'porcupine.db'))): time.sleep(0.02) else: self.replication_service = None # open items db while True: self._itemdb = db.DB(self._env) self._itemdb.set_pagesize(2048) try: self._itemdb.open('porcupine.db', 'items', dbtype=db.DB_BTREE, mode=db_mode, flags=db_flags) except db.DBLockDeadlockError: self._itemdb.close() continue break # open documents db while True: self._docdb = db.DB(self._env) try: self._docdb.open('porcupine.db', 'docs', dbtype=db.DB_HASH, mode=db_mode, flags=db_flags) except db.DBLockDeadlockError: self._docdb.close() continue break # open indices self._indices = {} for name, unique, immutable in settings['store']['indices']: self._indices[name] = DbIndex(self._env, self._itemdb, name, unique, immutable, db_flags) self._running = True maintain = kwargs.get('maintain', False) if maintain and self._maintenance_thread is None: # start deadlock detector self._maintenance_thread = Thread(target=self.__maintain, name='DB maintenance thread') self._maintenance_thread.start() # start checkpoint thread self._checkpoint_thread = Thread(target=self.__checkpoint, name='DB checkpoint thread') self._checkpoint_thread.start() #if hasattr(self._env, 'memp_trickle'): # # strart memp_trickle thread # self._trickle_thread = Thread(target=self.__trickle, # name='DB memp_trickle thread') # self._trickle_thread.start() def is_open(self): return self._running # item operations def get_item(self, oid): if type(oid) != bytes: oid = oid.encode('utf-8') try: return self._indices['_id'].db.get(oid, txn=context._trans and context._trans.txn) except UnicodeEncodeError: return None except (db.DBLockDeadlockError, db.DBLockNotGrantedError): if context._trans is not None: context._trans.abort() raise exceptions.DBRetryTransaction def put_item(self, item): try: self._itemdb.put( pack_value(item._pid) + b'_' + pack_value(item._id), persist.dumps(item), context._trans.txn) except (db.DBLockDeadlockError, db.DBLockNotGrantedError): context._trans.abort() raise exceptions.DBRetryTransaction except db.DBError as e: if e.args[0] == _err_unsupported_index_type: raise db.DBError('Unsupported indexed data type') else: raise def delete_item(self, item): try: self._itemdb.delete( pack_value(item._pid) + b'_' + pack_value(item._id), context._trans.txn) except (db.DBLockDeadlockError, db.DBLockNotGrantedError): context._trans.abort() raise exceptions.DBRetryTransaction # containers def get_children(self, container_id): cursor = Cursor(self._itemdb, '_pid') cursor.set_scope(container_id) cursor.set_range(None, None) return cursor def get_child_by_name(self, container_id, name): try: return self._indices['displayName'].db.get( pack_value(container_id) + b'_' + pack_value(name), txn=context._trans and context._trans.txn) except (db.DBLockDeadlockError, db.DBLockNotGrantedError): if context._trans is not None: context._trans.abort() raise exceptions.DBRetryTransaction # external attributes def get_external(self, id): try: return self._docdb.get(id.encode('ascii'), txn=context._trans and context._trans.txn) except (db.DBLockDeadlockError, db.DBLockNotGrantedError): if context._trans is not None: context._trans.abort() raise exceptions.DBRetryTransaction def put_external(self, id, stream): try: self._docdb.put(id.encode('ascii'), stream, context._trans.txn) except (db.DBLockDeadlockError, db.DBLockNotGrantedError): context._trans.abort() raise exceptions.DBRetryTransaction def delete_external(self, id): try: self._docdb.delete(id.encode('ascii'), context._trans.txn) except db.DBNotFoundError: # virtual external due to elastic schema pass except (db.DBLockDeadlockError, db.DBLockNotGrantedError): context._trans.abort() raise exceptions.DBRetryTransaction # indices def get_cursor_list(self, conditions): cur_list = [] for index, value in conditions: cursor = Cursor(self._indices[index].db, self._indices[index].name) if isinstance(value, (list, tuple)): reversed = (len(value) == 3 and value[2]) cursor.set_range(value[0], value[1]) if reversed: cursor.reverse() else: cursor.set(value) cur_list.append(cursor) return cur_list def query(self, conditions): cur_list = self.get_cursor_list(conditions) if len(cur_list) == 1: return cur_list[0] else: c_join = Join(self._itemdb, cur_list) return c_join def test_conditions(self, scope, conditions): cur_list = self.get_cursor_list(conditions) if len(cur_list) == 1: cursor = cur_list[0] else: cursor = Join(self._itemdb, cur_list) cursor.set_scope(scope) iterator = iter(cursor) try: result = bool(next(iterator)) except StopIteration: result = False cursor.close() return result # transactions def get_transaction(self, **kwargs): nosync = kwargs.get('nosync', False) snapshot = kwargs.get('snapshot', False) return Transaction(self._env, nosync, snapshot) def __remove_env(self): files = glob.glob(self.env_dir + '__db.*') for file in files: try: os.remove(file) except OSError: pass # administrative def __remove_files(self): # environment files self.__remove_env() # log files files = glob.glob(self.log_dir + 'log.*') for file in files: os.remove(file) # database file os.remove(self.dir + 'porcupine.db') # index file os.remove(self.dir + 'porcupine.idx') def truncate(self): # older versions of bsddb do not support truncate if hasattr(self._itemdb, 'truncate'): self._itemdb.truncate() self._docdb.truncate() else: # close database self.close() # remove old database files self.__remove_files() # open db self.__init__() def backup(self, output_file): # force checkpoint self._env.txn_checkpoint(0, 0, db.DB_FORCE) logs = self._env.log_archive(db.DB_ARCH_LOG) backfiles = [self.dir + 'porcupine.db', self.dir + 'porcupine.idx'] + \ [self.log_dir + log.decode() for log in logs] # compact backup.... backup = BackupFile(output_file) backup.add_files(backfiles) def restore(self, bset): self.__remove_files() backup = BackupFile(bset) backup.extract(self.dir, self.log_dir) def shrink(self): logs = self._env.log_archive() for log in logs: os.remove(self.log_dir + log) return len(logs) def __maintain(self): "deadlock detection thread" while self._running: time.sleep(0.01) # deadlock detection try: aborted = self._env.lock_detect(db.DB_LOCK_YOUNGEST) if aborted: logger.critical( "Deadlock: Aborted %d deadlocked transaction(s)" % aborted) except db.DBError: pass def __trickle(self): "memp_trickle thread" while self._running: self._env.memp_trickle(95) time.sleep(120) def __checkpoint(self): "checkpoint thread" while self._running: if self.replication_service is None \ or self.replication_service.is_master(): # checkpoint every 512KB written self._env.txn_checkpoint(512, 0) time.sleep(16) #stats = self._env.txn_stat() #print('txns: %d' % stats['nactive']) #print('max txns: %d' % stats['maxnactive']) #print() #stats = self._env.lock_stat() #print('Lockers: %d' % stats['nlockers']) #print('Max Lockers: %d' % stats['maxnlockers']) #print('Lockers wait: %d' % stats['lockers_wait']) #print() #print('Locks: %d' % stats['nlocks']) #print('Max Locks: %d' % stats['maxnlocks']) #print('Locks wait: %d' % stats['lock_wait']) #print('Locks no-wait: %d' % stats['lock_nowait']) #print() #print('Lock objects: %d' % stats['nobjects']) #print('Max objects: %d' % stats['maxnobjects']) #print('Objects wait: %d' % stats['objs_wait']) #print() #print('Requested: %d' % stats['nrequests']) #print('Released: %d' % stats['nreleases']) #print('-' * 80) def close(self, **kwargs): if self._running: self._running = False # join threads if self._maintenance_thread is not None: self._maintenance_thread.join() if self._checkpoint_thread is not None: self._checkpoint_thread.join() if self._trickle_thread is not None: self._trickle_thread.join() self._itemdb.close() self._docdb.close() # close indexes [index.close() for index in self._indices.values()] self._env.close() # clean-up environment files if (self._maintenance_thread is not None or kwargs.get('clear_env', False)): self.__remove_env()