def test_rtrie_in_mdb(self):
t = Trie()
self.assertEqual(t.add('hello'), 1)
self.assertEqual(t.add('hell'), 2)
self.assertEqual(t.add('hello'), 1)
self.assertEqual(t.add('hellothere'), 3)
self.assertEqual(t.add('good'), 4)
self.assertEqual(t.add('goodbye'), 5)
self.assertEqual(t.add('hello'), 1)
self.assertEqual(t.add('hellsink'), 6)
nodes, kids, _ = t.serialize()
nodeaddr, nodelen = nodes.buffer_info()
kidaddr, kidlen = kids.buffer_info()
try:
env = mdb.Env('/tmp/test_rtrie',
flags=mdb.MDB_WRITEMAP | mdb.MDB_NOSYNC
| mdb.MDB_NOSUBDIR)
txn = env.begin_txn()
db = env.open_db(txn, name='_meta_', flags=mdb.MDB_CREATE)
db.put_raw(txn, 'nodes', nodeaddr, nodelen)
db.put_raw(txn, 'kids', kidaddr, kidlen)
n, ns = db.get_raw(txn, 'nodes')
k, ks = db.get_raw(txn, 'kids')
txn.commit()
env.close()
env = mdb.Env('/tmp/test_rtrie',
flags=mdb.MDB_NOSYNC | mdb.MDB_NOSUBDIR)
txn = env.begin_txn()
db = env.open_db(txn, name='_meta_')
n, ns = db.get_raw(txn, 'nodes')
k, ks = db.get_raw(txn, 'kids')
self.assertEqual(rtrie.vid_for_value(n, k, 'hello'), 1)
self.assertEqual(rtrie.vid_for_value(n, k, 'hell'), 2)
self.assertEqual(rtrie.vid_for_value(n, k, 'goodbye'), 5)
self.assertEqual(rtrie.vid_for_value(n, k, 'hellsink'), 6)
self.assertEqual(rtrie.vid_for_value(n, k, 'hellothere'), 3)
self.assertEqual(rtrie.vid_for_value(n, k, 'good'), 4)
self.assertIsNone(rtrie.vid_for_value(n, k, 'notthere'))
txn.commit()
env.close()
finally:
import os
os.unlink('/tmp/test_rtrie')
os.unlink('/tmp/test_rtrie-lock')
def setUp(self):
import os
import errno
self.path = './testdbm'
try:
os.makedirs(self.path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(self.path):
pass
else:
raise
self.env = mdb.Env(self.path, mapsize=1 * mdb.MB, max_dbs=8)
def test_lru(self):
def get(db, txn, key):
try:
return db.get(txn, key)
except:
return None
env = mdb.Env('/tmp/lru_test',
flags=mdb.MDB_WRITEMAP | mdb.MDB_NOSYNC
| mdb.MDB_NOSUBDIR)
txn = env.begin_txn()
ixdb = env.open_db(txn, 'ix', flags=mdb.MDB_CREATE)
lru = LRUDict.getDict(5, partial(mdb_fetch, txn=txn, ixdb=ixdb),
partial(mdb_evict, txn=txn, ixdb=ixdb))
lru.set('hello', BitSet())
lru.set('goodbye', BitSet())
lru.set('amine', BitSet())
lru.set('solution', BitSet())
lru.set('lsd', BitSet())
self.assertEqual(len(lru._getContents()), 5)
lru.set('proxy', BitSet())
store = lru._getContents()
self.assertNotIn('hello', store)
self.assertIsNotNone(get(ixdb, txn, 'hello'))
self.assertEqual(len(store), 5)
bitmap = lru['hello']
store = lru._getContents()
self.assertIn('hello', store)
self.assertEqual(len(store), 5)
self.assertIsInstance(bitmap, BitSet)
self.assertIsNone(lru.get('skibiddles'))
# test eviction order
self.assertIsNotNone(
lru.get('goodbye')
) # this now should 'reset' goodbye so that it won't be evicted
lru.set('whammy bar', BitSet()) # amine should be evicted
store = lru._getContents()
self.assertNotIn('amine', store)
self.assertIn('goodbye', store)
txn.commit()
env.close()
def setUp(self):
import os
import errno
self.path = './testdbm'
try:
os.makedirs(self.path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(self.path):
pass
else:
raise
self.env = mdb.Env(self.path, max_dbs=8)
self.txn = self.env.begin_txn()
self.db = self.env.open_db(self.txn, 'test_cursor')
self.db.drop(self.txn, 0)
self.txn.commit()
self.txn = self.env.begin_txn()
def _open_env(cls, filename, maxsize, write):
# make sure we dont create a lock when opening for read
if write:
oflags = mdb.MDB_WRITEMAP
else:
oflags = mdb.MDB_RDONLY | mdb.MDB_NOLOCK
try:
env = mdb.Env(filename,
max_dbs=1024,
mapsize=maxsize,
flags=oflags | mdb.MDB_NOSYNC | mdb.MDB_NOSUBDIR)
except Exception as e:
import os
print "Exception: " + str(e)
for i in os.walk('/proc/' + str(os.getpid()) + '/fd/'):
print "XXX:", i[0], str(os.stat(i[0]))
raise e
return env
