def test_simple(): d = dict() lru = LRU(2, d) lru["x"] = 1 lru["y"] = 2 assert lru["x"] == 1 assert lru["y"] == 2 assert d == {"x": 1, "y": 2} lru["z"] = 3 assert len(d) == 2 assert len(lru) == 2 assert "z" in d assert "z" in lru assert "x" not in d assert "y" in d del lru["y"] assert "y" not in d assert "y" not in lru lru["a"] = 5 assert set(lru.keys()) == set(["z", "a"])
def test_simple(): d = dict() lru = LRU(2, d) lru['x'] = 1 lru['y'] = 2 assert lru['x'] == 1 assert lru['y'] == 2 assert d == {'x': 1, 'y': 2} lru['z'] = 3 assert len(d) == 2 assert len(lru) == 2 assert 'z' in d assert 'z' in lru assert 'x' not in d assert 'y' in d del lru['y'] assert 'y' not in d assert 'y' not in lru lru['a'] = 5 assert set(lru.keys()) == set(['z', 'a']) assert 'a' in str(lru) and '5' in str(lru) assert 'a' in repr(lru) and '5' in repr(lru)
def test_simple(): d = dict() lru = LRU(2, d) lru['x'] = 1 lru['y'] = 2 assert lru['x'] == 1 assert lru['y'] == 2 assert d == {'x': 1, 'y': 2} lru['z'] = 3 assert len(d) == 2 assert len(lru) == 2 assert 'z' in d assert 'z' in lru assert 'x' not in d assert 'y' in d del lru['y'] assert 'y' not in d assert 'y' not in lru lru['a'] = 5 assert set(lru.keys()) == set(['z', 'a'])
def test_explicit_evict(): d = dict() lru = LRU(10, d) lru['x'] = 1 lru['y'] = 2 assert set(d) == {'x', 'y'} lru.evict() assert set(d) == {'y'}
def __init__(self, upstream, maxsize=None, key=identity, hashable=True, **kwargs): self.key = key self.maxsize = maxsize if hashable: self.seen = dict() if self.maxsize: from zict import LRU self.seen = LRU(self.maxsize, self.seen) else: self.seen = [] Stream.__init__(self, upstream, **kwargs)
def test_overwrite(): d = dict() lru = LRU(2, d) lru["x"] = 1 lru["y"] = 2 lru["y"] = 3 assert set(lru) == {"x", "y"} lru.update({"y": 4}) assert set(lru) == {"x", "y"}
def test_overwrite(): d = dict() lru = LRU(2, d) lru['x'] = 1 lru['y'] = 2 lru['y'] = 3 assert set(lru) == {'x', 'y'} lru.update({'y': 4}) assert set(lru) == {'x', 'y'}
def test_explicit_evict(): d = dict() lru = LRU(10, d) lru["x"] = 1 lru["y"] = 2 assert set(d) == {"x", "y"} k, v, w = lru.evict() assert set(d) == {"y"} assert k == "x" assert v == 1 assert w == 1
def test_explicit_evict(): d = dict() lru = LRU(10, d) lru['x'] = 1 lru['y'] = 2 assert set(d) == {'x', 'y'} k, v, w = lru.evict() assert set(d) == {'y'} assert k == 'x' assert v == 1 assert w == 1
def __init__(self, child, history=None): self.seen = dict() if history: from zict import LRU self.seen = LRU(history, self.seen) Stream.__init__(self, child)
def __init__(self, upstream, history=None, key=identity, **kwargs): self.seen = dict() self.key = key if history: from zict import LRU self.seen = LRU(history, self.seen) Stream.__init__(self, upstream, **kwargs)
def __init__(self, child, history=None, key=identity): self.seen = dict() self.key = key if history: from zict import LRU self.seen = LRU(history, self.seen) Stream.__init__(self, child)
def test_mapping(): """ Test mapping interface for LRU(). """ d = {} # 100 is more than the max length when running check_mapping() lru = LRU(100, d) utils_test.check_mapping(lru) utils_test.check_closing(lru)
def test_callbacks(): L = list() d = dict() lru = LRU(2, d, on_evict=lambda k, v: L.append((k, v))) lru['x'] = 1 lru['y'] = 2 lru['z'] = 3 assert L == [('x', 1)]
def test_str(): d = dict() lru = LRU(2, d) lru["x"] = 1 lru["y"] = 2 assert str(lru.total_weight) in str(lru) assert str(lru.total_weight) in repr(lru) assert str(lru.n) in str(lru) assert str(lru.n) in repr(lru) assert "dict" in str(lru) assert "dict" in repr(lru)
def test_str(): d = dict() lru = LRU(2, d) lru['x'] = 1 lru['y'] = 2 assert str(lru.total_weight) in str(lru) assert str(lru.total_weight) in repr(lru) assert str(lru.n) in str(lru) assert str(lru.n) in repr(lru) assert 'dict' in str(lru) assert 'dict' in repr(lru)
def test_callbacks(): count = [0] def cb(k, v): count[0] += 1 L = list() d = dict() lru = LRU(2, d, on_evict=[lambda k, v: L.append((k, v)), cb]) lru['x'] = 1 lru['y'] = 2 lru['z'] = 3 assert L == [('x', 1)] assert count[0] == len(L)
def test_weight(): d = dict() weight = lambda k, v: v lru = LRU(10, d, weight=weight) lru["x"] = 5 assert lru.total_weight == 5 lru["y"] = 4 assert lru.total_weight == 9 lru["z"] = 3 assert d == {"y": 4, "z": 3} assert lru.total_weight == 7 del lru["z"] assert lru.total_weight == 4 lru["a"] = 10000 assert "a" not in lru assert d == {"y": 4}
def test_weight(): d = dict() weight = lambda k, v: v lru = LRU(10, d, weight=weight) lru['x'] = 5 assert lru.total_weight == 5 lru['y'] = 4 assert lru.total_weight == 9 lru['z'] = 3 assert d == {'y': 4, 'z': 3} assert lru.total_weight == 7 del lru['z'] assert lru.total_weight == 4 lru['a'] = 10000 assert 'a' not in lru assert d == {'y': 4}
def setup(self, worker): self.cache = Buffer( fast={}, slow=Func( dump=blosc.pack_array, load=blosc.unpack_array, d=Buffer( fast={}, slow=LRU( n=self._maxdisk, d=File(os.path.join(worker.local_directory, 'cache')), weight=lambda k, v: len(v), ), n=self._maxcompressed, weight=lambda k, v: len(v), ), ), n=self._maxmem, weight=lambda k, v: v.nbytes, ) self.lock = Lock() self.hits = 0 self.misses = 0
def setup(self, worker): self.cache = LRU(n=self._maxmem, d={}, weight=lambda k, v: v.nbytes) self.lock = Lock() self.hits = 0 self.misses = 0
class unique(Stream): """ Avoid sending through repeated elements This deduplicates a stream so that only new elements pass through. You can control how much of a history is stored with the ``maxsize=`` parameter. For example setting ``maxsize=1`` avoids sending through elements when one is repeated right after the other. Parameters ---------- maxsize: int or None, optional number of stored unique values to check against key : function, optional Function which returns a representation of the incoming data. For example ``key=lambda x: x['a']`` could be used to allow only pieces of data with unique ``'a'`` values to pass through. hashable : bool, optional If True then data is assumed to be hashable, else it is not. This is used for determining how to cache the history, if hashable then either dicts or LRU caches are used, otherwise a deque is used. Defaults to True. Examples -------- >>> source = Stream() >>> source.unique(maxsize=1).sink(print) >>> for x in [1, 1, 2, 2, 2, 1, 3]: ... source.emit(x) 1 2 1 3 """ def __init__(self, upstream, maxsize=None, key=identity, hashable=True, **kwargs): self.key = key self.maxsize = maxsize if hashable: self.seen = dict() if self.maxsize: from zict import LRU self.seen = LRU(self.maxsize, self.seen) else: self.seen = [] Stream.__init__(self, upstream, **kwargs) def update(self, x, who=None): y = self.key(x) emit = True if isinstance(self.seen, list): if y in self.seen: self.seen.remove(y) emit = False self.seen.insert(0, y) if self.maxsize: del self.seen[self.maxsize:] if emit: return self._emit(x) else: if self.seen.get(y, '~~not_seen~~') == '~~not_seen~~': self.seen[y] = 1 return self._emit(x)