Esempio n. 1
0
    def test_lru(self):
        def get(db, txn, key):
            try:
                return db.get(txn, key)
            except:
                return None

        env = mdb.Env('/tmp/lru_test',
                      flags=mdb.MDB_WRITEMAP | mdb.MDB_NOSYNC
                      | mdb.MDB_NOSUBDIR)
        txn = env.begin_txn()
        ixdb = env.open_db(txn, 'ix', flags=mdb.MDB_CREATE)

        lru = LRUDict.getDict(5, partial(mdb_fetch, txn=txn, ixdb=ixdb),
                              partial(mdb_evict, txn=txn, ixdb=ixdb))

        lru.set('hello', BitSet())
        lru.set('goodbye', BitSet())
        lru.set('amine', BitSet())
        lru.set('solution', BitSet())
        lru.set('lsd', BitSet())
        self.assertEqual(len(lru._getContents()), 5)

        lru.set('proxy', BitSet())
        store = lru._getContents()
        self.assertNotIn('hello', store)
        self.assertIsNotNone(get(ixdb, txn, 'hello'))
        self.assertEqual(len(store), 5)

        bitmap = lru['hello']
        store = lru._getContents()
        self.assertIn('hello', store)
        self.assertEqual(len(store), 5)
        self.assertIsInstance(bitmap, BitSet)
        self.assertIsNone(lru.get('skibiddles'))

        # test eviction order
        self.assertIsNotNone(
            lru.get('goodbye')
        )  # this now should 'reset' goodbye so that it won't be evicted
        lru.set('whammy bar', BitSet())  # amine should be evicted
        store = lru._getContents()
        self.assertNotIn('amine', store)
        self.assertIn('goodbye', store)

        txn.commit()
        env.close()
Esempio n. 2
0
    def test_lru(self):
        def get(db, txn, key):
            try:
                return db.get(txn, key)
            except:
                return None

        env = mdb.Env('/tmp/lru_test', flags=mdb.MDB_WRITEMAP | mdb.MDB_NOSYNC | mdb.MDB_NOSUBDIR)
        txn = env.begin_txn()
        ixdb = env.open_db(txn, 'ix', flags=mdb.MDB_CREATE)

        lru = LRUDict.getDict(5,
                      partial(mdb_fetch, txn=txn, ixdb=ixdb),
                      partial(mdb_evict, txn=txn, ixdb=ixdb))

        lru.set('hello', BitSet())
        lru.set('goodbye', BitSet())
        lru.set('amine', BitSet())
        lru.set('solution', BitSet())
        lru.set('lsd', BitSet())
        self.assertEqual(len(lru._getContents()), 5)

        lru.set('proxy', BitSet())
        store = lru._getContents()
        self.assertNotIn('hello', store)
        self.assertIsNotNone(get(ixdb, txn, 'hello'))
        self.assertEqual(len(store), 5)

        bitmap = lru['hello']
        store = lru._getContents()
        self.assertIn('hello', store)
        self.assertEqual(len(store), 5)
        self.assertIsInstance(bitmap, BitSet)
        self.assertIsNone(lru.get('skibiddles'))

        # test eviction order
        self.assertIsNotNone(lru.get('goodbye')) # this now should 'reset' goodbye so that it won't be evicted
        lru.set('whammy bar', BitSet()) # amine should be evicted
        store = lru._getContents()
        self.assertNotIn('amine', store)
        self.assertIn('goodbye', store)

        txn.commit()
        env.close()
Esempio n. 3
0
    def _open_dbs(self, env, write, lru_size):
        from pylru import LRUDict
        if write:
            txn = env.begin_txn()
        else:
            txn = env.begin_txn(flags=mdb.MDB_RDONLY)
        dbs = {}
        for index, column in self._columns.iteritems():
            subindexdb = None
            bitmap_dict = _dummy
            if column.index_indicator:
                # create an index for this column
                flags = mdb.MDB_CREATE
                if column.is_int:
                    flags |= mdb.MDB_INTEGERKEY
                subindexdb = env.open_db(txn,
                                         name="ix:%s" % index,
                                         flags=flags,
                                         key_inttype=column.get_effective_inttype())
                if write:
                    if column.index_indicator == 2:
                        evict = Victor(mdb_evict, txn, subindexdb)
                        fetch = Victor(mdb_fetch, txn, subindexdb)
                        bitmap_dict = LRUDict.getDict(lru_size,
                                                      fetch,
                                                      evict,
                                                      column.is_int,
                                                      BitSet)
                    else:
                        bitmap_dict = defaultdict(BitSet)

            flags = mdb.MDB_CREATE | mdb.MDB_INTEGERKEY
            if column.is_int:
                flags |= mdb.MDB_INTEGERDUP
            subdb = env.open_db(txn,
                                name=index,
                                flags=flags,
                                key_inttype=mdb.MDB_UINT_32,
                                value_inttype=column.get_effective_inttype())

            dbs[index] = (subdb, subindexdb, bitmap_dict, column)
        meta = env.open_db(txn, name='_meta_', flags=mdb.MDB_CREATE)
        return env, txn, dbs, meta
Esempio n. 4
0
    def _open_dbs(self, env, write, lru_size):
        from pylru import LRUDict
        if write:
            txn = env.begin_txn()
        else:
            txn = env.begin_txn(flags=mdb.MDB_RDONLY)
        dbs = {}
        for index, column in self._columns.iteritems():
            subindexdb = None
            bitmap_dict = _dummy
            if column.index_indicator:
                # create an index for this column
                flags = mdb.MDB_CREATE
                if column.is_int:
                    flags |= mdb.MDB_INTEGERKEY
                subindexdb = env.open_db(txn,
                                         name="ix:%s" % index,
                                         flags=flags,
                                         key_inttype=column.get_effective_inttype())
                if write:
                    if column.index_indicator == 2:
                        evict = Victor(mdb_evict, txn, subindexdb)
                        fetch = Victor(mdb_fetch, txn, subindexdb)
                        bitmap_dict = LRUDict.getDict(lru_size,
                                                      fetch,
                                                      evict,
                                                      column.is_int,
                                                      BitSet)
                    else:
                        bitmap_dict = defaultdict(BitSet)

            flags = mdb.MDB_CREATE | mdb.MDB_INTEGERKEY
            if column.is_int:
                flags |= mdb.MDB_INTEGERDUP
            subdb = env.open_db(txn,
                                name=index,
                                flags=flags,
                                key_inttype=mdb.MDB_UINT_32,
                                value_inttype=column.get_effective_inttype())

            dbs[index] = (subdb, subindexdb, bitmap_dict, column)
        meta = env.open_db(txn, name='_meta_', flags=mdb.MDB_CREATE)
        return env, txn, dbs, meta