def test_simple():
d = dict()
lru = LRU(2, d)
lru["x"] = 1
lru["y"] = 2
assert lru["x"] == 1
assert lru["y"] == 2
assert d == {"x": 1, "y": 2}
lru["z"] = 3
assert len(d) == 2
assert len(lru) == 2
assert "z" in d
assert "z" in lru
assert "x" not in d
assert "y" in d
del lru["y"]
assert "y" not in d
assert "y" not in lru
lru["a"] = 5
assert set(lru.keys()) == set(["z", "a"])
def test_simple():
d = dict()
lru = LRU(2, d)
lru['x'] = 1
lru['y'] = 2
assert lru['x'] == 1
assert lru['y'] == 2
assert d == {'x': 1, 'y': 2}
lru['z'] = 3
assert len(d) == 2
assert len(lru) == 2
assert 'z' in d
assert 'z' in lru
assert 'x' not in d
assert 'y' in d
del lru['y']
assert 'y' not in d
assert 'y' not in lru
lru['a'] = 5
assert set(lru.keys()) == set(['z', 'a'])
assert 'a' in str(lru) and '5' in str(lru)
assert 'a' in repr(lru) and '5' in repr(lru)
def test_simple():
d = dict()
lru = LRU(2, d)
lru['x'] = 1
lru['y'] = 2
assert lru['x'] == 1
assert lru['y'] == 2
assert d == {'x': 1, 'y': 2}
lru['z'] = 3
assert len(d) == 2
assert len(lru) == 2
assert 'z' in d
assert 'z' in lru
assert 'x' not in d
assert 'y' in d
del lru['y']
assert 'y' not in d
assert 'y' not in lru
lru['a'] = 5
assert set(lru.keys()) == set(['z', 'a'])
def test_explicit_evict():
d = dict()
lru = LRU(10, d)
lru['x'] = 1
lru['y'] = 2
assert set(d) == {'x', 'y'}
lru.evict()
assert set(d) == {'y'}
def __init__(self, upstream, maxsize=None, key=identity, hashable=True,
**kwargs):
self.key = key
self.maxsize = maxsize
if hashable:
self.seen = dict()
if self.maxsize:
from zict import LRU
self.seen = LRU(self.maxsize, self.seen)
else:
self.seen = []
Stream.__init__(self, upstream, **kwargs)
def test_overwrite():
d = dict()
lru = LRU(2, d)
lru["x"] = 1
lru["y"] = 2
lru["y"] = 3
assert set(lru) == {"x", "y"}
lru.update({"y": 4})
assert set(lru) == {"x", "y"}
def test_overwrite():
d = dict()
lru = LRU(2, d)
lru['x'] = 1
lru['y'] = 2
lru['y'] = 3
assert set(lru) == {'x', 'y'}
lru.update({'y': 4})
assert set(lru) == {'x', 'y'}
def test_overwrite():
d = dict()
lru = LRU(2, d)
lru['x'] = 1
lru['y'] = 2
lru['y'] = 3
assert set(lru) == {'x', 'y'}
lru.update({'y': 4})
assert set(lru) == {'x', 'y'}
def test_explicit_evict():
d = dict()
lru = LRU(10, d)
lru["x"] = 1
lru["y"] = 2
assert set(d) == {"x", "y"}
k, v, w = lru.evict()
assert set(d) == {"y"}
assert k == "x"
assert v == 1
assert w == 1
def test_explicit_evict():
d = dict()
lru = LRU(10, d)
lru['x'] = 1
lru['y'] = 2
assert set(d) == {'x', 'y'}
k, v, w = lru.evict()
assert set(d) == {'y'}
assert k == 'x'
assert v == 1
assert w == 1
def __init__(self, child, history=None):
self.seen = dict()
if history:
from zict import LRU
self.seen = LRU(history, self.seen)
Stream.__init__(self, child)
def __init__(self, upstream, history=None, key=identity, **kwargs):
self.seen = dict()
self.key = key
if history:
from zict import LRU
self.seen = LRU(history, self.seen)
Stream.__init__(self, upstream, **kwargs)
def __init__(self, child, history=None, key=identity):
self.seen = dict()
self.key = key
if history:
from zict import LRU
self.seen = LRU(history, self.seen)
Stream.__init__(self, child)
def test_mapping():
"""
Test mapping interface for LRU().
"""
d = {}
# 100 is more than the max length when running check_mapping()
lru = LRU(100, d)
utils_test.check_mapping(lru)
utils_test.check_closing(lru)
def test_callbacks():
L = list()
d = dict()
lru = LRU(2, d, on_evict=lambda k, v: L.append((k, v)))
lru['x'] = 1
lru['y'] = 2
lru['z'] = 3
assert L == [('x', 1)]
def test_str():
d = dict()
lru = LRU(2, d)
lru["x"] = 1
lru["y"] = 2
assert str(lru.total_weight) in str(lru)
assert str(lru.total_weight) in repr(lru)
assert str(lru.n) in str(lru)
assert str(lru.n) in repr(lru)
assert "dict" in str(lru)
assert "dict" in repr(lru)
def test_str():
d = dict()
lru = LRU(2, d)
lru['x'] = 1
lru['y'] = 2
assert str(lru.total_weight) in str(lru)
assert str(lru.total_weight) in repr(lru)
assert str(lru.n) in str(lru)
assert str(lru.n) in repr(lru)
assert 'dict' in str(lru)
assert 'dict' in repr(lru)
def test_callbacks():
count = [0]
def cb(k, v):
count[0] += 1
L = list()
d = dict()
lru = LRU(2, d, on_evict=[lambda k, v: L.append((k, v)), cb])
lru['x'] = 1
lru['y'] = 2
lru['z'] = 3
assert L == [('x', 1)]
assert count[0] == len(L)
def test_weight():
d = dict()
weight = lambda k, v: v
lru = LRU(10, d, weight=weight)
lru["x"] = 5
assert lru.total_weight == 5
lru["y"] = 4
assert lru.total_weight == 9
lru["z"] = 3
assert d == {"y": 4, "z": 3}
assert lru.total_weight == 7
del lru["z"]
assert lru.total_weight == 4
lru["a"] = 10000
assert "a" not in lru
assert d == {"y": 4}
def test_weight():
d = dict()
weight = lambda k, v: v
lru = LRU(10, d, weight=weight)
lru['x'] = 5
assert lru.total_weight == 5
lru['y'] = 4
assert lru.total_weight == 9
lru['z'] = 3
assert d == {'y': 4, 'z': 3}
assert lru.total_weight == 7
del lru['z']
assert lru.total_weight == 4
lru['a'] = 10000
assert 'a' not in lru
assert d == {'y': 4}
def setup(self, worker):
self.cache = Buffer(
fast={},
slow=Func(
dump=blosc.pack_array,
load=blosc.unpack_array,
d=Buffer(
fast={},
slow=LRU(
n=self._maxdisk,
d=File(os.path.join(worker.local_directory, 'cache')),
weight=lambda k, v: len(v),
),
n=self._maxcompressed,
weight=lambda k, v: len(v),
),
),
n=self._maxmem,
weight=lambda k, v: v.nbytes,
)
self.lock = Lock()
self.hits = 0
self.misses = 0
def setup(self, worker):
self.cache = LRU(n=self._maxmem, d={}, weight=lambda k, v: v.nbytes)
self.lock = Lock()
self.hits = 0
self.misses = 0
class unique(Stream):
""" Avoid sending through repeated elements
This deduplicates a stream so that only new elements pass through.
You can control how much of a history is stored with the ``maxsize=``
parameter. For example setting ``maxsize=1`` avoids sending through
elements when one is repeated right after the other.
Parameters
----------
maxsize: int or None, optional
number of stored unique values to check against
key : function, optional
Function which returns a representation of the incoming data.
For example ``key=lambda x: x['a']`` could be used to allow only
pieces of data with unique ``'a'`` values to pass through.
hashable : bool, optional
If True then data is assumed to be hashable, else it is not. This is
used for determining how to cache the history, if hashable then
either dicts or LRU caches are used, otherwise a deque is used.
Defaults to True.
Examples
--------
>>> source = Stream()
>>> source.unique(maxsize=1).sink(print)
>>> for x in [1, 1, 2, 2, 2, 1, 3]:
... source.emit(x)
1
2
1
3
"""
def __init__(self,
upstream,
maxsize=None,
key=identity,
hashable=True,
**kwargs):
self.key = key
self.maxsize = maxsize
if hashable:
self.seen = dict()
if self.maxsize:
from zict import LRU
self.seen = LRU(self.maxsize, self.seen)
else:
self.seen = []
Stream.__init__(self, upstream, **kwargs)
def update(self, x, who=None):
y = self.key(x)
emit = True
if isinstance(self.seen, list):
if y in self.seen:
self.seen.remove(y)
emit = False
self.seen.insert(0, y)
if self.maxsize:
del self.seen[self.maxsize:]
if emit:
return self._emit(x)
else:
if self.seen.get(y, '~~not_seen~~') == '~~not_seen~~':
self.seen[y] = 1
return self._emit(x)
