def open(self, name, txn, **kwds): self._threaded = threading.local() self._db = DB(self.store.env) self._db.set_lorder(4321) self._flags = 0 if kwds.get('ramdb', False): name = None dbname = None else: dbname = kwds.get('dbname') if kwds.get('create', False): self._db.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_CREATE | DB_THREAD | self._flags, txn = txn) else: self._db.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_THREAD | self._flags, txn = txn) super(DBContainer, self).__init__(self._db)
def main(cls, argv): if len(argv) != 2: print "Usage: BerkeleyDbSearcher <index dir>" return dbHome = argv[1] env = DBEnv() env.set_flags(DB_LOG_INMEMORY, 1); if os.name == 'nt': env.set_cachesize(0, 0x4000000, 1) elif os.name == 'posix': from commands import getstatusoutput if getstatusoutput('uname') == (0, 'Linux'): env.set_cachesize(0, 0x4000000, 1) env.open(dbHome, (DB_THREAD | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0) index = DB(env) blocks = DB(env) txn = None try: txn = env.txn_begin(None) index.open(filename = '__index__', dbtype = DB_BTREE, flags = DB_THREAD, txn = txn) blocks.open(filename = '__blocks__', dbtype = DB_BTREE, flags = DB_THREAD, txn = txn) except: if txn is not None: txn.abort() txn = None raise else: txn.commit() txn = None try: txn = env.txn_begin(None) directory = DbDirectory(txn, index, blocks, 0) searcher = self.getSearcher() topDocs = searcher.search(TermQuery(Term("contents", "fox")), 50) print topDocs.totalHits, "document(s) found" del searcher except: if txn is not None: txn.abort() txn = None raise else: txn.abort() index.close() blocks.close() env.close()
def openIndex(self, name, dbname, txn, keyMethod, **kwds): if kwds.get('ramdb', False): name = None dbname = None index = DB(self.store.env) index.set_lorder(4321) if kwds.get('create', False): index.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_CREATE | DB_THREAD | self._flags, txn = txn) else: index.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_THREAD | self._flags, txn = txn) self._db.associate(secondaryDB = index, callback = keyMethod, flags = 0, txn = txn) return index
class DBContainer(object): def __init__(self, store): self.store = store self._db = None def open(self, name, txn, **kwds): self._threaded = threading.local() self._db = DB(self.store.env) self._db.set_lorder(4321) self._flags = 0 if kwds.get('ramdb', False): name = None dbname = None else: dbname = kwds.get('dbname') if kwds.get('create', False): self._db.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_CREATE | DB_THREAD | self._flags, txn = txn) else: self._db.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_THREAD | self._flags, txn = txn) super(DBContainer, self).__init__(self._db) def openIndex(self, name, dbname, txn, keyMethod, **kwds): if kwds.get('ramdb', False): name = None dbname = None index = DB(self.store.env) index.set_lorder(4321) if kwds.get('create', False): index.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_CREATE | DB_THREAD | self._flags, txn = txn) else: index.open(filename = name, dbname = dbname, dbtype = DB_BTREE, flags = DB_THREAD | self._flags, txn = txn) self._db.associate(secondaryDB = index, callback = keyMethod, flags = 0, txn = txn) return index def close(self): if self._db is not None: self._db.close() self._db = None self._threaded = None def attachView(self, view): pass def detachView(self, view): pass def put(self, key, value): self._db.put(key, value, self.store.txn) return len(key) + len(value) def delete(self, key): try: self._db.delete(key, self.store.txn) except DBNotFoundError: pass def get(self, key): while True: try: return self._db.get(key, None, self.store.txn, self._flags) except DBLockDeadlockError: self._logDL(24) if self.store.txn is not None: raise def openCursor(self, db=None): if db is None: db = self._db try: cursor = self._threaded.cursors[db].dup() self.store.repository.logger.info('duplicated cursor') return cursor except AttributeError: self._threaded.cursors = {} except KeyError: pass cursor = db.cursor(self.store.txn, self._flags) self._threaded.cursors[db] = cursor return cursor def closeCursor(self, cursor, db=None): if cursor is not None: if db is None: db = self._db try: if self._threaded.cursors[db] is cursor: del self._threaded.cursors[db] except KeyError: pass cursor.close() def _logDL(self, n): self.store.repository.logger.info('detected deadlock: %d', n) def _readValue(self, value, offset): code = value[offset] offset += 1 if code == '\0': return (1, None) if code == '\1': return (1, True) if code == '\2': return (1, False) if code == '\3': return (17, UUID(value[offset:offset+16])) if code == '\4': return (5, unpack('>l', value[offset:offset+4])[0]) if code == '\5': l, = unpack('>H', value[offset:offset+2]) offset += 2 return (l + 3, value[offset:offset+l]) raise ValueError, code def _writeUUID(self, buffer, value): if value is None: buffer.write('\0') else: buffer.write('\3') buffer.write(value._uuid) def _writeString(self, buffer, value): if value is None: buffer.write('\0') elif isinstance(value, str): buffer.write('\5') buffer.write(pack('>H', len(value))) buffer.write(value) elif isinstance(value, unicode): value = value.encode('utf-8') buffer.write('\5') buffer.write(pack('>H', len(value))) buffer.write(value) else: raise TypeError, type(value) def _writeBoolean(self, buffer, value): if value is True: buffer.write('\1') elif value is False: buffer.write('\2') else: raise TypeError, type(value) def _writeInteger(self, buffer, value): if value is None: buffer.write('\0') buffer.write('\4') buffer.write(pack('>l', value)) def _writeValue(self, buffer, value): if value is None: buffer.write('\0') elif value is True or value is False: self._writeBoolean(buffer, value) elif isinstance(value, str) or isinstance(value, unicode): self._writeString(buffer, value) elif isinstance(value, int) or isinstance(value, long): self._writeInteger(buffer, value) elif isinstance(value, UUID): self._writeUUID(buffer, value) else: raise NotImplementedError, "value: %s, type: %s" %(value, type(value))
def main(cls, argv): if len(argv) < 2: print "Usage: BerkeleyDbIndexer <index dir> -create" return dbHome = argv[1] create = len(argv) > 2 and argv[2] == "-create" if not os.path.exists(dbHome): os.makedirs(dbHome) elif create: for name in os.listdir(dbHome): if name.startswith('__'): os.remove(os.path.join(dbHome, name)) env = DBEnv() env.set_flags(DB_LOG_INMEMORY, 1); if os.name == 'nt': env.set_cachesize(0, 0x4000000, 1) elif os.name == 'posix': from commands import getstatusoutput if getstatusoutput('uname') == (0, 'Linux'): env.set_cachesize(0, 0x4000000, 1) env.open(dbHome, (DB_CREATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0) index = DB(env) blocks = DB(env) txn = None try: txn = env.txn_begin(None) index.open(filename = '__index__', dbtype = DB_BTREE, flags = DB_CREATE | DB_THREAD, txn = txn) blocks.open(filename = '__blocks__', dbtype = DB_BTREE, flags = DB_CREATE | DB_THREAD, txn = txn) except: if txn is not None: txn.abort() txn = None raise else: txn.commit() txn = None try: txn = env.txn_begin(None) directory = DbDirectory(txn, index, blocks, 0) writer = IndexWriter(directory, StandardAnalyzer(), create, IndexWriter.MaxFieldLength.UNLIMITED) writer.setUseCompoundFile(False) doc = Document() doc.add(Field("contents", "The quick brown fox...", Field.Store.YES, Field.Index.ANALYZED)) writer.addDocument(doc) writer.commit() writer.close() except: if txn is not None: txn.abort() txn = None raise else: txn.commit() index.close() blocks.close() env.close() print "Indexing Complete"
def main(cls, argv): if len(argv) < 2: print "Usage: BerkeleyDbIndexer <index dir> -create" return dbHome = argv[1] create = len(argv) > 2 and argv[2] == "-create" if not os.path.exists(dbHome): os.makedirs(dbHome) elif create: for name in os.listdir(dbHome): if name.startswith('__'): os.remove(os.path.join(dbHome, name)) env = DBEnv() env.set_flags(DB_LOG_INMEMORY, 1) if os.name == 'nt': env.set_cachesize(0, 0x4000000, 1) elif os.name == 'posix': from commands import getstatusoutput if getstatusoutput('uname') == (0, 'Linux'): env.set_cachesize(0, 0x4000000, 1) env.open(dbHome, (DB_CREATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0) index = DB(env) blocks = DB(env) txn = None try: txn = env.txn_begin(None) index.open(filename='__index__', dbtype=DB_BTREE, flags=DB_CREATE | DB_THREAD, txn=txn) blocks.open(filename='__blocks__', dbtype=DB_BTREE, flags=DB_CREATE | DB_THREAD, txn=txn) except: if txn is not None: txn.abort() txn = None raise else: txn.commit() txn = None try: txn = env.txn_begin(None) directory = DbDirectory(txn, index, blocks, 0) writer = IndexWriter(directory, StandardAnalyzer(), create, IndexWriter.MaxFieldLength.UNLIMITED) writer.setUseCompoundFile(False) doc = Document() doc.add( Field("contents", "The quick brown fox...", Field.Store.YES, Field.Index.ANALYZED)) writer.addDocument(doc) writer.optimize() writer.close() except: if txn is not None: txn.abort() txn = None raise else: txn.commit() index.close() blocks.close() env.close() print "Indexing Complete"
def main(cls, argv): if len(argv) != 2: print "Usage: BerkeleyDbSearcher <index dir>" return dbHome = argv[1] env = DBEnv() env.set_flags(DB_LOG_INMEMORY, 1); if os.name == 'nt': env.set_cachesize(0, 0x4000000, 1) elif os.name == 'posix': from commands import getstatusoutput if getstatusoutput('uname') == (0, 'Linux'): env.set_cachesize(0, 0x4000000, 1) env.open(dbHome, (DB_THREAD | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0) index = DB(env) blocks = DB(env) txn = None try: txn = env.txn_begin(None) index.open(filename = '__index__', dbtype = DB_BTREE, flags = DB_THREAD, txn = txn) blocks.open(filename = '__blocks__', dbtype = DB_BTREE, flags = DB_THREAD, txn = txn) except: if txn is not None: txn.abort() txn = None raise else: txn.commit() txn = None try: txn = env.txn_begin(None) directory = DbDirectory(txn, index, blocks, 0) searcher = IndexSearcher(directory, True) topDocs = searcher.search(TermQuery(Term("contents", "fox")), 50) print topDocs.totalHits, "document(s) found" searcher.close() except: if txn is not None: txn.abort() txn = None raise else: txn.abort() index.close() blocks.close() env.close()