def setUp(self):
self.store = acid.open('ListEngine')
self.store.add_collection('stuff')
self.keys = [
self.store['stuff'].put('a'),
self.store['stuff'].put('b'),
self.store['stuff'].put('c')
]
def setUp(self):
self.store = acid.open('list:/')
self.txn = self.store.begin()
self.txn.__enter__()
self.store.add_collection('stuff')
self.keys = [
self.store['stuff'].put(u'a'), self.store['stuff'].put(u'b'),
self.store['stuff'].put(u'c')
]
def setUp(self):
self.store = acid.open('list:/')
self.txn = self.store.begin()
self.txn.__enter__()
self.store.add_collection('stuff')
self.keys = [
self.store['stuff'].put(u'a'),
self.store['stuff'].put(u'b'),
self.store['stuff'].put(u'c')
]
def test1(self):
store = acid.open('list:/')
with store.begin(write=True):
stuff = store.add_collection('stuff')
acid.add_index(stuff, 'foop', lambda rec: 'foop')
key = stuff.put(u'temp')
assert store['foop'].find('foop') == 'temp'
assert stuff.get(1) == 'temp'
stuff.delete(1)
assert stuff.get(1) is None
i = store['foop'].find('foop')
assert i is None, i
def test1(self):
store = acid.open('list:/')
with store.begin(write=True):
stuff = store.add_collection('stuff')
acid.add_index(stuff, 'foop', lambda rec: 'foop')
key = stuff.put(u'temp')
assert store['foop'].find('foop') == 'temp'
assert stuff.get(1) == 'temp'
stuff.delete(1)
assert stuff.get(1) is None
i = store['foop'].find('foop')
assert i is None, i
def setUp(self):
self.store = acid.open('ListEngine')
self.store.add_collection('stuff')
self.keys = [
self.store['stuff'].put('a'),
self.store['stuff'].put('b'),
self.store['stuff'].put('c')
]
self.store.add_collection('stuff0')
self.store['stuff0'].put('a')
self.store['stuff0'].put('b')
self.store['stuff0'].put('c')
OneCollBoundsTest.setUp(self)
self.store.add_collection('stuff2')
self.store['stuff2'].put('a')
self.store['stuff2'].put('b')
self.store['stuff2'].put('c')
def main():
parser = make_parser()
opts, args = parser.parse_args()
if not args:
die('Please specify a command (see --help)')
if not opts.db:
die('Please specify store URL (-d, --db)')
global STORE
STORE = acid.open(opts.db)
signal.signal(signal.SIGWINCH, _on_sigwinch)
_on_sigwinch()
func = globals().get('cmd_' + args[0])
if not func:
die('No such command: %r' % (args[0],))
func(opts, args[1:])
def main():
parser = make_parser()
opts, args = parser.parse_args()
if not args:
die('Please specify a command (see --help)')
if not opts.db:
die('Please specify store URL (-d, --db)')
global STORE
STORE = acid.open(opts.db)
signal.signal(signal.SIGWINCH, _on_sigwinch)
_on_sigwinch()
func = globals().get('cmd_' + args[0])
if not func:
die('No such command: %r' % (args[0], ))
func(opts, args[1:])
def setUp(self):
self.store = acid.open('ListEngine')
self.e = self.store.engine
self.coll = self.store.add_collection('stuff')
self.i = self.coll.add_index('idx', lambda obj: (69, obj))
self.key = self.coll.put('dave')
self.key2 = self.coll.put('dave2')
self.expect = [(69, 'dave'), self.key]
self.expect2 = [(69, 'dave2'), self.key2]
self.first = [self.expect]
self.second = [self.expect2]
self.both = [self.expect, self.expect2]
# Insert junk in a higher collection to test iter stop conds.
self.coll2 = self.store.add_collection('stuff2')
self.i2 = self.coll2.add_index('idx', lambda obj: (69, obj))
self.coll2.put('XXXX')
self.coll2.put('YYYY')
def setUp(self):
self.store = acid.open('list:/')
self.e = self.store.engine
self.t = self.store.begin()
self.t.__enter__()
self.coll = self.store.add_collection('stuff')
self.i = acid.add_index(self.coll, 'idx', lambda obj: (69, obj))
self.key = self.coll.put(u'dave')
self.key2 = self.coll.put(u'dave2')
self.expect = [(69, u'dave'), self.key]
self.expect2 = [(69, u'dave2'), self.key2]
self.first = [self.expect]
self.second = [self.expect2]
self.both = [self.expect, self.expect2]
# Insert junk in a higher collection to test iter stop conds.
self.coll2 = self.store.add_collection('stuff2')
self.i2 = acid.add_index(self.coll2, 'idx', lambda obj: (69, obj))
self.coll2.put(u'XXXX')
self.coll2.put(u'YYYY')
def setUp(self):
self.store = acid.open('list:/')
self.e = self.store.engine
self.t = self.store.begin()
self.t.__enter__()
self.coll = self.store.add_collection('stuff')
self.i = acid.add_index(self.coll, 'idx', lambda obj: (69, obj))
self.key = self.coll.put(u'dave')
self.key2 = self.coll.put(u'dave2')
self.expect = [(69, u'dave'), self.key]
self.expect2 = [(69, u'dave2'), self.key2]
self.first = [self.expect]
self.second = [self.expect2]
self.both = [self.expect, self.expect2]
# Insert junk in a higher collection to test iter stop conds.
self.coll2 = self.store.add_collection('stuff2')
self.i2 = acid.add_index(self.coll2, 'idx', lambda obj: (69, obj))
self.coll2.put(u'XXXX')
self.coll2.put(u'YYYY')
parent_id = acid.meta.Integer()
@acid.meta.constraint
def check_id(self):
return self.id is not None
@acid.meta.key
def key(self):
key = [self.id]
parent_id = self.parent_id
while parent_id:
key.append(parent_id)
parent = self.get(id=parent_id)
assert parent
parent_id = parent.id
key.reverse()
return tuple(key)
@acid.meta.index
def first_last(self):
return self.first_name, self.last_name
store = acid.open('ListEngine')
MyModel.bind_store(store)
i = Item(id=123)
print i
i.save()
print i
def setUp(self):
self.store = acid.open('list:/')
def make_engine(self):
self.store = acid.open('LmdbEngine',
path=self.PATH,
map_size=1048576 * 1024,
writemap=USE_SPARSE_FILES)
def make_engine(self):
self.store = acid.open('PlyvelEngine', name=self.PATH,
create_if_missing=True)
import mailbox
import acid
import acid.encoders
store = acid.open('LmdbEngine', path='/tmp/email.lmdb', map_size=1048576000000,
writemap=True)
coll = store.add_collection('emails')
def bat():
last_key = ()
while last_key is not None:
txn = store.engine.begin(write=True)
found, made, last_key = coll.batch(
lo=last_key, txn=txn, max_bytes=32767,
packer=acid.encoders.ZLIB_PACKER)
txn.commit()
print 'found=', found, 'made=', made
'''
mbox = mailbox.Maildir('/home/backup/eldil/mail/botanicus')
it = iter(mbox)
env = store.engine.env
idx = 0
def make_engine(self):
self.store = acid.open('LmdbEngine',
path=self.PATH, map_size=1048576*1024,
writemap=USE_SPARSE_FILES)
def make_engine(self):
self.store = acid.open('PlyvelEngine',
name=self.PATH,
create_if_missing=True)
def engine_count(engine):
return sum(1 for _ in engine.iter('', False))
def engine_size(engine):
return sum(len(k) + len(v) for k, v in engine.iter('', False))
for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
if not packer:
continue
print
for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
if os.path.exists(TMP_PATH):
shutil.rmtree(TMP_PATH)
st = acid.open('LmdbEngine', path=TMP_PATH, map_size=1048576*1024)
co = st.add_collection('people',
encoder=acid.encoders.make_json_encoder(sort_keys=True))
keys = [co.put(rec) for rec in recs]
random.shuffle(keys)
nextkey = iter(itertools.cycle(keys)).next
before = engine_size(st.engine)
itemcount = engine_count(st.engine)
if bsize == 1:
iterrecs, te = dotestiter()
out('plain', before / 1024., itemcount, 1, 1, dotestget(), te, iterrecs)
co.batch(max_recs=bsize, packer=packer)
def store_size(store):
it = store._txn_context.get().iter("", False)
return sum(len(k) + len(v) for k, v in it)
for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
if not packer:
continue
print
for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
if os.path.exists(TMP_PATH):
shutil.rmtree(TMP_PATH)
st = acid.open("lmdb:%s" % (TMP_PATH,))
with st.begin(write=True):
co = st.add_collection("people", encoder=acid.encoders.make_json_encoder(sort_keys=True), compressor=packer)
keys = [co.put(rec) for rec in recs]
random.shuffle(keys)
nextkey = iter(itertools.cycle(keys)).next
before = store_size(st)
itemcount = store_len(st)
if bsize == 1:
iterrecs, te = dotestiter()
out("plain", before / 1024.0, itemcount, 1, 1, dotestget(), te, iterrecs)
co.strategy.batch(max_recs=bsize)
def setUp(self):
self.store = acid.open("list:/")
Model.bind_store(self.store)
def setUp(self):
self.store = acid.open('list:/')
Model.bind_store(self.store)
def store_size(store):
it = store._txn_context.get().iter('', False)
return sum(len(k) + len(v) for k, v in it)
for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
if not packer:
continue
print
for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
if os.path.exists(TMP_PATH):
shutil.rmtree(TMP_PATH)
st = acid.open('lmdb:%s' % (TMP_PATH, ))
with st.begin(write=True):
co = st.add_collection(
'people',
encoder=acid.encoders.make_json_encoder(sort_keys=True),
compressor=packer)
keys = [co.put(rec) for rec in recs]
random.shuffle(keys)
nextkey = iter(itertools.cycle(keys)).next
before = store_size(st)
itemcount = store_len(st)
if bsize == 1:
iterrecs, te = dotestiter()
def make_engine(self):
self.store = acid.open('SkiplistEngine', maxsize=int(1e9))
import mailbox
import acid
import acid.encoders
store = acid.open('LmdbEngine',
path='/tmp/email.lmdb',
map_size=1048576000000,
writemap=True)
coll = store.add_collection('emails')
def bat():
last_key = ()
while last_key is not None:
txn = store.engine.begin(write=True)
found, made, last_key = coll.batch(lo=last_key,
txn=txn,
max_bytes=32767,
packer=acid.encoders.ZLIB_PACKER)
txn.commit()
print 'found=', found, 'made=', made
'''
mbox = mailbox.Maildir('/home/backup/eldil/mail/botanicus')
it = iter(mbox)
env = store.engine.env
idx = 0
while True:
def setUp(self):
self.store = acid.open('ListEngine')
Model.bind_store(self.store)
rand = range(1, 1 + len(words))
random.shuffle(rand)
def rands(coll, keys):
t = time.time()
for k in keys:
assert coll.get(k, raw=True) is not None, [k]
t1 = time.time()
return len(keys) / (t1 - t)
for strat_klass in acid.core.BatchV2Strategy, acid.core.BatchStrategy, :
compressor = acid.encoders.ZLIB
store = acid.open('list:/')
store.begin(write=True).__enter__()
doink = store.add_collection('doink')
prefix = acid.keylib.pack_int(doink.info['idx'], store.prefix)
strat = strat_klass(prefix, store, compressor)
doink.strategy = strat
for word in words:
doink.put(word)
print 'done', strat, compressor
print 'before len:', store_len(store)
print 'before size:', store_size(store)
print 'avgsz:', store_size(store) / store_len(store)
print 'look/sec', rands(doink, rand)
print
rand = range(1, 1+len(words))
random.shuffle(rand)
def rands(coll, keys):
t = time.time()
for k in keys:
assert coll.get(k, raw=True) is not None, [k]
t1 = time.time()
return len(keys) / (t1 - t)
for strat_klass in acid.core.BatchV2Strategy, acid.core.BatchStrategy, :
compressor = acid.encoders.ZLIB
store = acid.open('list:/')
store.begin(write=True).__enter__()
doink = store.add_collection('doink')
prefix = acid.keylib.pack_int(doink.info['idx'], store.prefix)
strat = strat_klass(prefix, store, compressor)
doink.strategy = strat
for word in words:
doink.put(word)
print 'done', strat, compressor
print 'before len:', store_len(store)
print 'before size:', store_size(store)
print 'avgsz:', store_size(store)/store_len(store)
print 'look/sec', rands(doink, rand)
print
def setUp(self):
self.store = acid.open('list:/')
def make_engine(self):
self.store = acid.open('skiplist:/')