print
for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
if os.path.exists(TMP_PATH):
shutil.rmtree(TMP_PATH)
st = acid.open('lmdb:%s' % (TMP_PATH, ))
with st.begin(write=True):
co = st.add_collection(
'people',
encoder=acid.encoders.make_json_encoder(sort_keys=True),
compressor=packer)
keys = [co.put(rec) for rec in recs]
random.shuffle(keys)
nextkey = iter(itertools.cycle(keys)).next
before = store_size(st)
itemcount = store_len(st)
if bsize == 1:
iterrecs, te = dotestiter()
out('plain', before / 1024., itemcount, 1, 1, dotestget(), te,
iterrecs)
co.strategy.batch(max_recs=bsize)
itemcount = store_len(st)
after = store_len(st)
iterrecs, te = dotestiter()
out(packer.name, after / 1024., itemcount,
float(before) / after, bsize, dotestget(), te, iterrecs)
for strat_klass in acid.core.BatchV2Strategy, acid.core.BatchStrategy, :
compressor = acid.encoders.ZLIB
store = acid.open('list:/')
store.begin(write=True).__enter__()
doink = store.add_collection('doink')
prefix = acid.keylib.pack_int(doink.info['idx'], store.prefix)
strat = strat_klass(prefix, store, compressor)
doink.strategy = strat
for word in words:
doink.put(word)
print 'done', strat, compressor
print 'before len:', store_len(store)
print 'before size:', store_size(store)
print 'avgsz:', store_size(store) / store_len(store)
print 'look/sec', rands(doink, rand)
print
strat.batch(max_bytes=2000)
print 'done', strat, compressor
print 'after len:', store_len(store)
print 'after size:', store_size(store)
print 'avgsz:', store_size(store) / store_len(store)
print 'look/sec', rands(doink, rand)
print
li = store.engine.items[-1]
for strat_klass in acid.core.BatchV2Strategy, acid.core.BatchStrategy, :
compressor = acid.encoders.ZLIB
store = acid.open('list:/')
store.begin(write=True).__enter__()
doink = store.add_collection('doink')
prefix = acid.keylib.pack_int(doink.info['idx'], store.prefix)
strat = strat_klass(prefix, store, compressor)
doink.strategy = strat
for word in words:
doink.put(word)
print 'done', strat, compressor
print 'before len:', store_len(store)
print 'before size:', store_size(store)
print 'avgsz:', store_size(store)/store_len(store)
print 'look/sec', rands(doink, rand)
print
strat.batch(max_bytes=2000)
print 'done', strat, compressor
print 'after len:', store_len(store)
print 'after size:', store_size(store)
print 'avgsz:', store_size(store)/store_len(store)
print 'look/sec', rands(doink, rand)
print
li = store.engine.items[-1]
for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
if not packer:
continue
print
for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
if os.path.exists(TMP_PATH):
shutil.rmtree(TMP_PATH)
st = acid.open("lmdb:%s" % (TMP_PATH,))
with st.begin(write=True):
co = st.add_collection("people", encoder=acid.encoders.make_json_encoder(sort_keys=True), compressor=packer)
keys = [co.put(rec) for rec in recs]
random.shuffle(keys)
nextkey = iter(itertools.cycle(keys)).next
before = store_size(st)
itemcount = store_len(st)
if bsize == 1:
iterrecs, te = dotestiter()
out("plain", before / 1024.0, itemcount, 1, 1, dotestget(), te, iterrecs)
co.strategy.batch(max_recs=bsize)
itemcount = store_len(st)
after = store_len(st)
iterrecs, te = dotestiter()
out(packer.name, after / 1024.0, itemcount, float(before) / after, bsize, dotestget(), te, iterrecs)
