示例#1
0
 def setUp(self):
     self.store = acid.open('ListEngine')
     self.store.add_collection('stuff')
     self.keys = [
         self.store['stuff'].put('a'),
         self.store['stuff'].put('b'),
         self.store['stuff'].put('c')
     ]
示例#2
0
 def setUp(self):
     self.store = acid.open('list:/')
     self.txn = self.store.begin()
     self.txn.__enter__()
     self.store.add_collection('stuff')
     self.keys = [
         self.store['stuff'].put(u'a'), self.store['stuff'].put(u'b'),
         self.store['stuff'].put(u'c')
     ]
示例#3
0
文件: core_test.py 项目: dw/acid
 def setUp(self):
     self.store = acid.open('list:/')
     self.txn = self.store.begin()
     self.txn.__enter__()
     self.store.add_collection('stuff')
     self.keys = [
         self.store['stuff'].put(u'a'),
         self.store['stuff'].put(u'b'),
         self.store['stuff'].put(u'c')
     ]
示例#4
0
文件: core_test.py 项目: dw/acid
 def test1(self):
     store = acid.open('list:/')
     with store.begin(write=True):
         stuff = store.add_collection('stuff')
         acid.add_index(stuff, 'foop', lambda rec: 'foop')
         key = stuff.put(u'temp')
         assert store['foop'].find('foop') == 'temp'
         assert stuff.get(1) == 'temp'
         stuff.delete(1)
         assert stuff.get(1) is None
         i = store['foop'].find('foop')
         assert i is None, i
示例#5
0
 def test1(self):
     store = acid.open('list:/')
     with store.begin(write=True):
         stuff = store.add_collection('stuff')
         acid.add_index(stuff, 'foop', lambda rec: 'foop')
         key = stuff.put(u'temp')
         assert store['foop'].find('foop') == 'temp'
         assert stuff.get(1) == 'temp'
         stuff.delete(1)
         assert stuff.get(1) is None
         i = store['foop'].find('foop')
         assert i is None, i
示例#6
0
 def setUp(self):
     self.store = acid.open('ListEngine')
     self.store.add_collection('stuff')
     self.keys = [
         self.store['stuff'].put('a'),
         self.store['stuff'].put('b'),
         self.store['stuff'].put('c')
     ]
     self.store.add_collection('stuff0')
     self.store['stuff0'].put('a')
     self.store['stuff0'].put('b')
     self.store['stuff0'].put('c')
     OneCollBoundsTest.setUp(self)
     self.store.add_collection('stuff2')
     self.store['stuff2'].put('a')
     self.store['stuff2'].put('b')
     self.store['stuff2'].put('c')
示例#7
0
文件: tool.py 项目: dw/acid
def main():
    parser = make_parser()
    opts, args = parser.parse_args()

    if not args:
        die('Please specify a command (see --help)')
    if not opts.db:
        die('Please specify store URL (-d, --db)')

    global STORE
    STORE = acid.open(opts.db)
    signal.signal(signal.SIGWINCH, _on_sigwinch)
    _on_sigwinch()

    func = globals().get('cmd_' + args[0])
    if not func:
        die('No such command: %r' % (args[0],))

    func(opts, args[1:])
示例#8
0
def main():
    parser = make_parser()
    opts, args = parser.parse_args()

    if not args:
        die('Please specify a command (see --help)')
    if not opts.db:
        die('Please specify store URL (-d, --db)')

    global STORE
    STORE = acid.open(opts.db)
    signal.signal(signal.SIGWINCH, _on_sigwinch)
    _on_sigwinch()

    func = globals().get('cmd_' + args[0])
    if not func:
        die('No such command: %r' % (args[0], ))

    func(opts, args[1:])
示例#9
0
    def setUp(self):
        self.store = acid.open('ListEngine')
        self.e = self.store.engine
        self.coll = self.store.add_collection('stuff')
        self.i = self.coll.add_index('idx', lambda obj: (69, obj))

        self.key = self.coll.put('dave')
        self.key2 = self.coll.put('dave2')

        self.expect = [(69, 'dave'), self.key]
        self.expect2 = [(69, 'dave2'), self.key2]
        self.first = [self.expect]
        self.second = [self.expect2]
        self.both = [self.expect, self.expect2]

        # Insert junk in a higher collection to test iter stop conds.
        self.coll2 = self.store.add_collection('stuff2')
        self.i2 = self.coll2.add_index('idx', lambda obj: (69, obj))
        self.coll2.put('XXXX')
        self.coll2.put('YYYY')
示例#10
0
文件: core_test.py 项目: dw/acid
    def setUp(self):
        self.store = acid.open('list:/')
        self.e = self.store.engine
        self.t = self.store.begin()
        self.t.__enter__()
        self.coll = self.store.add_collection('stuff')
        self.i = acid.add_index(self.coll, 'idx', lambda obj: (69, obj))

        self.key = self.coll.put(u'dave')
        self.key2 = self.coll.put(u'dave2')

        self.expect = [(69, u'dave'), self.key]
        self.expect2 = [(69, u'dave2'), self.key2]
        self.first = [self.expect]
        self.second = [self.expect2]
        self.both = [self.expect, self.expect2]

        # Insert junk in a higher collection to test iter stop conds.
        self.coll2 = self.store.add_collection('stuff2')
        self.i2 = acid.add_index(self.coll2, 'idx', lambda obj: (69, obj))
        self.coll2.put(u'XXXX')
        self.coll2.put(u'YYYY')
示例#11
0
    def setUp(self):
        self.store = acid.open('list:/')
        self.e = self.store.engine
        self.t = self.store.begin()
        self.t.__enter__()
        self.coll = self.store.add_collection('stuff')
        self.i = acid.add_index(self.coll, 'idx', lambda obj: (69, obj))

        self.key = self.coll.put(u'dave')
        self.key2 = self.coll.put(u'dave2')

        self.expect = [(69, u'dave'), self.key]
        self.expect2 = [(69, u'dave2'), self.key2]
        self.first = [self.expect]
        self.second = [self.expect2]
        self.both = [self.expect, self.expect2]

        # Insert junk in a higher collection to test iter stop conds.
        self.coll2 = self.store.add_collection('stuff2')
        self.i2 = acid.add_index(self.coll2, 'idx', lambda obj: (69, obj))
        self.coll2.put(u'XXXX')
        self.coll2.put(u'YYYY')
示例#12
0
    parent_id = acid.meta.Integer()

    @acid.meta.constraint
    def check_id(self):
        return self.id is not None

    @acid.meta.key
    def key(self):
        key = [self.id]
        parent_id = self.parent_id
        while parent_id:
            key.append(parent_id)
            parent = self.get(id=parent_id)
            assert parent
            parent_id = parent.id
        key.reverse()
        return tuple(key)

    @acid.meta.index
    def first_last(self):
        return self.first_name, self.last_name


store = acid.open('ListEngine')
MyModel.bind_store(store)

i = Item(id=123)
print i
i.save()
print i
示例#13
0
 def setUp(self):
     self.store = acid.open('list:/')
示例#14
0
文件: benchy.py 项目: lazerhawk/acid
 def make_engine(self):
     self.store = acid.open('LmdbEngine',
                            path=self.PATH,
                            map_size=1048576 * 1024,
                            writemap=USE_SPARSE_FILES)
示例#15
0
文件: benchy.py 项目: prologic/acid
 def make_engine(self):
     self.store = acid.open('PlyvelEngine', name=self.PATH,
                               create_if_missing=True)
示例#16
0
import mailbox

import acid
import acid.encoders



store = acid.open('LmdbEngine', path='/tmp/email.lmdb', map_size=1048576000000,
                     writemap=True)
coll = store.add_collection('emails')



def bat():
    last_key = ()
    while last_key is not None:
        txn = store.engine.begin(write=True)
        found, made, last_key = coll.batch(
            lo=last_key, txn=txn, max_bytes=32767,
            packer=acid.encoders.ZLIB_PACKER)
        txn.commit()
        print 'found=', found, 'made=', made


'''
mbox = mailbox.Maildir('/home/backup/eldil/mail/botanicus')
it = iter(mbox)
env = store.engine.env
idx = 0
示例#17
0
文件: benchy.py 项目: prologic/acid
 def make_engine(self):
     self.store = acid.open('LmdbEngine',
         path=self.PATH, map_size=1048576*1024,
         writemap=USE_SPARSE_FILES)
示例#18
0
文件: benchy.py 项目: dw/acid
 def make_engine(self):
     self.store = acid.open('skiplist:/')
示例#19
0
文件: benchy.py 项目: lazerhawk/acid
 def make_engine(self):
     self.store = acid.open('PlyvelEngine',
                            name=self.PATH,
                            create_if_missing=True)
示例#20
0
文件: batch.py 项目: prologic/acid
def engine_count(engine):
    return sum(1 for _ in engine.iter('', False))

def engine_size(engine):
    return sum(len(k) + len(v) for k, v in engine.iter('', False))


for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
    if not packer:
        continue

    print
    for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
        if os.path.exists(TMP_PATH):
            shutil.rmtree(TMP_PATH)
        st = acid.open('LmdbEngine', path=TMP_PATH, map_size=1048576*1024)
        co = st.add_collection('people',
            encoder=acid.encoders.make_json_encoder(sort_keys=True))

        keys = [co.put(rec) for rec in recs]
        random.shuffle(keys)
        nextkey = iter(itertools.cycle(keys)).next

        before = engine_size(st.engine)
        itemcount = engine_count(st.engine)

        if bsize == 1:
            iterrecs, te = dotestiter()
            out('plain', before / 1024., itemcount, 1, 1, dotestget(), te, iterrecs)
        co.batch(max_recs=bsize, packer=packer)
示例#21
0
文件: batch.py 项目: pombredanne/acid

def store_size(store):
    it = store._txn_context.get().iter("", False)
    return sum(len(k) + len(v) for k, v in it)


for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
    if not packer:
        continue

    print
    for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
        if os.path.exists(TMP_PATH):
            shutil.rmtree(TMP_PATH)
        st = acid.open("lmdb:%s" % (TMP_PATH,))
        with st.begin(write=True):
            co = st.add_collection("people", encoder=acid.encoders.make_json_encoder(sort_keys=True), compressor=packer)

            keys = [co.put(rec) for rec in recs]
            random.shuffle(keys)
            nextkey = iter(itertools.cycle(keys)).next

            before = store_size(st)
            itemcount = store_len(st)

            if bsize == 1:
                iterrecs, te = dotestiter()
                out("plain", before / 1024.0, itemcount, 1, 1, dotestget(), te, iterrecs)
            co.strategy.batch(max_recs=bsize)
示例#22
0
 def setUp(self):
     self.store = acid.open("list:/")
     Model.bind_store(self.store)
示例#23
0
 def setUp(self):
     self.store = acid.open('list:/')
     Model.bind_store(self.store)
示例#24
0

def store_size(store):
    it = store._txn_context.get().iter('', False)
    return sum(len(k) + len(v) for k, v in it)


for packer in acid.encoders.ZLIB, SNAPPY, LZ4:
    if not packer:
        continue

    print
    for bsize in 1, 2, 4, 5, 8, 16, 32, 64:
        if os.path.exists(TMP_PATH):
            shutil.rmtree(TMP_PATH)
        st = acid.open('lmdb:%s' % (TMP_PATH, ))
        with st.begin(write=True):
            co = st.add_collection(
                'people',
                encoder=acid.encoders.make_json_encoder(sort_keys=True),
                compressor=packer)

            keys = [co.put(rec) for rec in recs]
            random.shuffle(keys)
            nextkey = iter(itertools.cycle(keys)).next

            before = store_size(st)
            itemcount = store_len(st)

            if bsize == 1:
                iterrecs, te = dotestiter()
示例#25
0
文件: benchy.py 项目: prologic/acid
 def make_engine(self):
      self.store = acid.open('SkiplistEngine', maxsize=int(1e9))
示例#26
0
import mailbox

import acid
import acid.encoders

store = acid.open('LmdbEngine',
                  path='/tmp/email.lmdb',
                  map_size=1048576000000,
                  writemap=True)
coll = store.add_collection('emails')


def bat():
    last_key = ()
    while last_key is not None:
        txn = store.engine.begin(write=True)
        found, made, last_key = coll.batch(lo=last_key,
                                           txn=txn,
                                           max_bytes=32767,
                                           packer=acid.encoders.ZLIB_PACKER)
        txn.commit()
        print 'found=', found, 'made=', made


'''
mbox = mailbox.Maildir('/home/backup/eldil/mail/botanicus')
it = iter(mbox)
env = store.engine.env
idx = 0

while True:
示例#27
0
 def setUp(self):
     self.store = acid.open('ListEngine')
     Model.bind_store(self.store)
示例#28
0
rand = range(1, 1 + len(words))
random.shuffle(rand)


def rands(coll, keys):
    t = time.time()
    for k in keys:
        assert coll.get(k, raw=True) is not None, [k]
    t1 = time.time()
    return len(keys) / (t1 - t)


for strat_klass in acid.core.BatchV2Strategy, acid.core.BatchStrategy, :
    compressor = acid.encoders.ZLIB

    store = acid.open('list:/')
    store.begin(write=True).__enter__()
    doink = store.add_collection('doink')
    prefix = acid.keylib.pack_int(doink.info['idx'], store.prefix)
    strat = strat_klass(prefix, store, compressor)
    doink.strategy = strat

    for word in words:
        doink.put(word)

    print 'done', strat, compressor
    print 'before len:', store_len(store)
    print 'before size:', store_size(store)
    print 'avgsz:', store_size(store) / store_len(store)
    print 'look/sec', rands(doink, rand)
    print
示例#29
0
文件: smallbatch.py 项目: dw/acid
rand = range(1, 1+len(words))
random.shuffle(rand)


def rands(coll, keys):
    t = time.time()
    for k in keys:
        assert coll.get(k, raw=True) is not None, [k]
    t1 = time.time()
    return len(keys) / (t1 - t)


for strat_klass in acid.core.BatchV2Strategy, acid.core.BatchStrategy, :
    compressor = acid.encoders.ZLIB

    store = acid.open('list:/')
    store.begin(write=True).__enter__()
    doink = store.add_collection('doink')
    prefix = acid.keylib.pack_int(doink.info['idx'], store.prefix)
    strat = strat_klass(prefix, store, compressor)
    doink.strategy = strat

    for word in words:
        doink.put(word)

    print 'done', strat, compressor
    print 'before len:', store_len(store)
    print 'before size:', store_size(store)
    print 'avgsz:', store_size(store)/store_len(store)
    print 'look/sec', rands(doink, rand)
    print
示例#30
0
文件: core_test.py 项目: dw/acid
 def setUp(self):
     self.store = acid.open('list:/')
示例#31
0
文件: benchy.py 项目: lazerhawk/acid
 def make_engine(self):
     self.store = acid.open('skiplist:/')