def open(self, name, txn, **kwds):
self._threaded = threading.local()
self._db = DB(self.store.env)
self._db.set_lorder(4321)
self._flags = 0
if kwds.get('ramdb', False):
name = None
dbname = None
else:
dbname = kwds.get('dbname')
if kwds.get('create', False):
self._db.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_CREATE | DB_THREAD | self._flags,
txn = txn)
else:
self._db.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_THREAD | self._flags,
txn = txn)
super(DBContainer, self).__init__(self._db)
def main(cls, argv):
if len(argv) != 2:
print "Usage: BerkeleyDbSearcher <index dir>"
return
dbHome = argv[1]
env = DBEnv()
env.set_flags(DB_LOG_INMEMORY, 1);
if os.name == 'nt':
env.set_cachesize(0, 0x4000000, 1)
elif os.name == 'posix':
from commands import getstatusoutput
if getstatusoutput('uname') == (0, 'Linux'):
env.set_cachesize(0, 0x4000000, 1)
env.open(dbHome, (DB_THREAD |
DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0)
index = DB(env)
blocks = DB(env)
txn = None
try:
txn = env.txn_begin(None)
index.open(filename = '__index__', dbtype = DB_BTREE,
flags = DB_THREAD, txn = txn)
blocks.open(filename = '__blocks__', dbtype = DB_BTREE,
flags = DB_THREAD, txn = txn)
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.commit()
txn = None
try:
txn = env.txn_begin(None)
directory = DbDirectory(txn, index, blocks, 0)
searcher = self.getSearcher()
topDocs = searcher.search(TermQuery(Term("contents", "fox")), 50)
print topDocs.totalHits, "document(s) found"
del searcher
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.abort()
index.close()
blocks.close()
env.close()
def openIndex(self, name, dbname, txn, keyMethod, **kwds):
if kwds.get('ramdb', False):
name = None
dbname = None
index = DB(self.store.env)
index.set_lorder(4321)
if kwds.get('create', False):
index.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_CREATE | DB_THREAD | self._flags,
txn = txn)
else:
index.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_THREAD | self._flags,
txn = txn)
self._db.associate(secondaryDB = index,
callback = keyMethod,
flags = 0,
txn = txn)
return index
class DBContainer(object):
def __init__(self, store):
self.store = store
self._db = None
def open(self, name, txn, **kwds):
self._threaded = threading.local()
self._db = DB(self.store.env)
self._db.set_lorder(4321)
self._flags = 0
if kwds.get('ramdb', False):
name = None
dbname = None
else:
dbname = kwds.get('dbname')
if kwds.get('create', False):
self._db.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_CREATE | DB_THREAD | self._flags,
txn = txn)
else:
self._db.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_THREAD | self._flags,
txn = txn)
super(DBContainer, self).__init__(self._db)
def openIndex(self, name, dbname, txn, keyMethod, **kwds):
if kwds.get('ramdb', False):
name = None
dbname = None
index = DB(self.store.env)
index.set_lorder(4321)
if kwds.get('create', False):
index.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_CREATE | DB_THREAD | self._flags,
txn = txn)
else:
index.open(filename = name, dbname = dbname,
dbtype = DB_BTREE,
flags = DB_THREAD | self._flags,
txn = txn)
self._db.associate(secondaryDB = index,
callback = keyMethod,
flags = 0,
txn = txn)
return index
def close(self):
if self._db is not None:
self._db.close()
self._db = None
self._threaded = None
def attachView(self, view):
pass
def detachView(self, view):
pass
def put(self, key, value):
self._db.put(key, value, self.store.txn)
return len(key) + len(value)
def delete(self, key):
try:
self._db.delete(key, self.store.txn)
except DBNotFoundError:
pass
def get(self, key):
while True:
try:
return self._db.get(key, None, self.store.txn, self._flags)
except DBLockDeadlockError:
self._logDL(24)
if self.store.txn is not None:
raise
def openCursor(self, db=None):
if db is None:
db = self._db
try:
cursor = self._threaded.cursors[db].dup()
self.store.repository.logger.info('duplicated cursor')
return cursor
except AttributeError:
self._threaded.cursors = {}
except KeyError:
pass
cursor = db.cursor(self.store.txn, self._flags)
self._threaded.cursors[db] = cursor
return cursor
def closeCursor(self, cursor, db=None):
if cursor is not None:
if db is None:
db = self._db
try:
if self._threaded.cursors[db] is cursor:
del self._threaded.cursors[db]
except KeyError:
pass
cursor.close()
def _logDL(self, n):
self.store.repository.logger.info('detected deadlock: %d', n)
def _readValue(self, value, offset):
code = value[offset]
offset += 1
if code == '\0':
return (1, None)
if code == '\1':
return (1, True)
if code == '\2':
return (1, False)
if code == '\3':
return (17, UUID(value[offset:offset+16]))
if code == '\4':
return (5, unpack('>l', value[offset:offset+4])[0])
if code == '\5':
l, = unpack('>H', value[offset:offset+2])
offset += 2
return (l + 3, value[offset:offset+l])
raise ValueError, code
def _writeUUID(self, buffer, value):
if value is None:
buffer.write('\0')
else:
buffer.write('\3')
buffer.write(value._uuid)
def _writeString(self, buffer, value):
if value is None:
buffer.write('\0')
elif isinstance(value, str):
buffer.write('\5')
buffer.write(pack('>H', len(value)))
buffer.write(value)
elif isinstance(value, unicode):
value = value.encode('utf-8')
buffer.write('\5')
buffer.write(pack('>H', len(value)))
buffer.write(value)
else:
raise TypeError, type(value)
def _writeBoolean(self, buffer, value):
if value is True:
buffer.write('\1')
elif value is False:
buffer.write('\2')
else:
raise TypeError, type(value)
def _writeInteger(self, buffer, value):
if value is None:
buffer.write('\0')
buffer.write('\4')
buffer.write(pack('>l', value))
def _writeValue(self, buffer, value):
if value is None:
buffer.write('\0')
elif value is True or value is False:
self._writeBoolean(buffer, value)
elif isinstance(value, str) or isinstance(value, unicode):
self._writeString(buffer, value)
elif isinstance(value, int) or isinstance(value, long):
self._writeInteger(buffer, value)
elif isinstance(value, UUID):
self._writeUUID(buffer, value)
else:
raise NotImplementedError, "value: %s, type: %s" %(value,
type(value))
def main(cls, argv):
if len(argv) < 2:
print "Usage: BerkeleyDbIndexer <index dir> -create"
return
dbHome = argv[1]
create = len(argv) > 2 and argv[2] == "-create"
if not os.path.exists(dbHome):
os.makedirs(dbHome)
elif create:
for name in os.listdir(dbHome):
if name.startswith('__'):
os.remove(os.path.join(dbHome, name))
env = DBEnv()
env.set_flags(DB_LOG_INMEMORY, 1);
if os.name == 'nt':
env.set_cachesize(0, 0x4000000, 1)
elif os.name == 'posix':
from commands import getstatusoutput
if getstatusoutput('uname') == (0, 'Linux'):
env.set_cachesize(0, 0x4000000, 1)
env.open(dbHome, (DB_CREATE | DB_THREAD |
DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0)
index = DB(env)
blocks = DB(env)
txn = None
try:
txn = env.txn_begin(None)
index.open(filename = '__index__', dbtype = DB_BTREE,
flags = DB_CREATE | DB_THREAD, txn = txn)
blocks.open(filename = '__blocks__', dbtype = DB_BTREE,
flags = DB_CREATE | DB_THREAD, txn = txn)
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.commit()
txn = None
try:
txn = env.txn_begin(None)
directory = DbDirectory(txn, index, blocks, 0)
writer = IndexWriter(directory, StandardAnalyzer(), create,
IndexWriter.MaxFieldLength.UNLIMITED)
writer.setUseCompoundFile(False)
doc = Document()
doc.add(Field("contents", "The quick brown fox...",
Field.Store.YES, Field.Index.ANALYZED))
writer.addDocument(doc)
writer.commit()
writer.close()
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.commit()
index.close()
blocks.close()
env.close()
print "Indexing Complete"
def main(cls, argv):
if len(argv) < 2:
print "Usage: BerkeleyDbIndexer <index dir> -create"
return
dbHome = argv[1]
create = len(argv) > 2 and argv[2] == "-create"
if not os.path.exists(dbHome):
os.makedirs(dbHome)
elif create:
for name in os.listdir(dbHome):
if name.startswith('__'):
os.remove(os.path.join(dbHome, name))
env = DBEnv()
env.set_flags(DB_LOG_INMEMORY, 1)
if os.name == 'nt':
env.set_cachesize(0, 0x4000000, 1)
elif os.name == 'posix':
from commands import getstatusoutput
if getstatusoutput('uname') == (0, 'Linux'):
env.set_cachesize(0, 0x4000000, 1)
env.open(dbHome, (DB_CREATE | DB_THREAD | DB_INIT_MPOOL | DB_INIT_LOCK
| DB_INIT_TXN), 0)
index = DB(env)
blocks = DB(env)
txn = None
try:
txn = env.txn_begin(None)
index.open(filename='__index__',
dbtype=DB_BTREE,
flags=DB_CREATE | DB_THREAD,
txn=txn)
blocks.open(filename='__blocks__',
dbtype=DB_BTREE,
flags=DB_CREATE | DB_THREAD,
txn=txn)
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.commit()
txn = None
try:
txn = env.txn_begin(None)
directory = DbDirectory(txn, index, blocks, 0)
writer = IndexWriter(directory, StandardAnalyzer(), create,
IndexWriter.MaxFieldLength.UNLIMITED)
writer.setUseCompoundFile(False)
doc = Document()
doc.add(
Field("contents", "The quick brown fox...", Field.Store.YES,
Field.Index.ANALYZED))
writer.addDocument(doc)
writer.optimize()
writer.close()
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.commit()
index.close()
blocks.close()
env.close()
print "Indexing Complete"
def main(cls, argv):
if len(argv) != 2:
print "Usage: BerkeleyDbSearcher <index dir>"
return
dbHome = argv[1]
env = DBEnv()
env.set_flags(DB_LOG_INMEMORY, 1);
if os.name == 'nt':
env.set_cachesize(0, 0x4000000, 1)
elif os.name == 'posix':
from commands import getstatusoutput
if getstatusoutput('uname') == (0, 'Linux'):
env.set_cachesize(0, 0x4000000, 1)
env.open(dbHome, (DB_THREAD |
DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN), 0)
index = DB(env)
blocks = DB(env)
txn = None
try:
txn = env.txn_begin(None)
index.open(filename = '__index__', dbtype = DB_BTREE,
flags = DB_THREAD, txn = txn)
blocks.open(filename = '__blocks__', dbtype = DB_BTREE,
flags = DB_THREAD, txn = txn)
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.commit()
txn = None
try:
txn = env.txn_begin(None)
directory = DbDirectory(txn, index, blocks, 0)
searcher = IndexSearcher(directory, True)
topDocs = searcher.search(TermQuery(Term("contents", "fox")), 50)
print topDocs.totalHits, "document(s) found"
searcher.close()
except:
if txn is not None:
txn.abort()
txn = None
raise
else:
txn.abort()
index.close()
blocks.close()
env.close()
