Beispiel #1
0
def test_simple():
    d = dict()
    lru = LRU(2, d)

    lru["x"] = 1
    lru["y"] = 2

    assert lru["x"] == 1
    assert lru["y"] == 2
    assert d == {"x": 1, "y": 2}

    lru["z"] = 3
    assert len(d) == 2
    assert len(lru) == 2
    assert "z" in d
    assert "z" in lru
    assert "x" not in d
    assert "y" in d

    del lru["y"]
    assert "y" not in d
    assert "y" not in lru

    lru["a"] = 5
    assert set(lru.keys()) == set(["z", "a"])
Beispiel #2
0
def test_simple():
    d = dict()
    lru = LRU(2, d)

    lru['x'] = 1
    lru['y'] = 2

    assert lru['x'] == 1
    assert lru['y'] == 2
    assert d == {'x': 1, 'y': 2}

    lru['z'] = 3
    assert len(d) == 2
    assert len(lru) == 2
    assert 'z' in d
    assert 'z' in lru
    assert 'x' not in d
    assert 'y' in d

    del lru['y']
    assert 'y' not in d
    assert 'y' not in lru

    lru['a'] = 5
    assert set(lru.keys()) == set(['z', 'a'])

    assert 'a' in str(lru) and '5' in str(lru)
    assert 'a' in repr(lru) and '5' in repr(lru)
Beispiel #3
0
def test_simple():
    d = dict()
    lru = LRU(2, d)

    lru['x'] = 1
    lru['y'] = 2

    assert lru['x'] == 1
    assert lru['y'] == 2
    assert d == {'x': 1, 'y': 2}

    lru['z'] = 3
    assert len(d) == 2
    assert len(lru) == 2
    assert 'z' in d
    assert 'z' in lru
    assert 'x' not in d
    assert 'y' in d

    del lru['y']
    assert 'y' not in d
    assert 'y' not in lru

    lru['a'] = 5
    assert set(lru.keys()) == set(['z', 'a'])
Beispiel #4
0
def test_explicit_evict():
    d = dict()
    lru = LRU(10, d)

    lru['x'] = 1
    lru['y'] = 2

    assert set(d) == {'x', 'y'}

    lru.evict()
    assert set(d) == {'y'}
Beispiel #5
0
    def __init__(self, upstream, maxsize=None, key=identity, hashable=True,
                 **kwargs):
        self.key = key
        self.maxsize = maxsize
        if hashable:
            self.seen = dict()
            if self.maxsize:
                from zict import LRU
                self.seen = LRU(self.maxsize, self.seen)
        else:
            self.seen = []

        Stream.__init__(self, upstream, **kwargs)
Beispiel #6
0
def test_overwrite():
    d = dict()
    lru = LRU(2, d)

    lru["x"] = 1
    lru["y"] = 2
    lru["y"] = 3

    assert set(lru) == {"x", "y"}

    lru.update({"y": 4})

    assert set(lru) == {"x", "y"}
Beispiel #7
0
def test_overwrite():
    d = dict()
    lru = LRU(2, d)

    lru['x'] = 1
    lru['y'] = 2
    lru['y'] = 3

    assert set(lru) == {'x', 'y'}

    lru.update({'y': 4})

    assert set(lru) == {'x', 'y'}
Beispiel #8
0
def test_overwrite():
    d = dict()
    lru = LRU(2, d)

    lru['x'] = 1
    lru['y'] = 2
    lru['y'] = 3

    assert set(lru) == {'x', 'y'}

    lru.update({'y': 4})

    assert set(lru) == {'x', 'y'}
Beispiel #9
0
def test_explicit_evict():
    d = dict()
    lru = LRU(10, d)

    lru["x"] = 1
    lru["y"] = 2

    assert set(d) == {"x", "y"}

    k, v, w = lru.evict()
    assert set(d) == {"y"}
    assert k == "x"
    assert v == 1
    assert w == 1
Beispiel #10
0
def test_explicit_evict():
    d = dict()
    lru = LRU(10, d)

    lru['x'] = 1
    lru['y'] = 2

    assert set(d) == {'x', 'y'}

    k, v, w = lru.evict()
    assert set(d) == {'y'}
    assert k == 'x'
    assert v == 1
    assert w == 1
Beispiel #11
0
    def __init__(self, child, history=None):
        self.seen = dict()
        if history:
            from zict import LRU
            self.seen = LRU(history, self.seen)

        Stream.__init__(self, child)
Beispiel #12
0
    def __init__(self, upstream, history=None, key=identity, **kwargs):
        self.seen = dict()
        self.key = key
        if history:
            from zict import LRU
            self.seen = LRU(history, self.seen)

        Stream.__init__(self, upstream, **kwargs)
Beispiel #13
0
    def __init__(self, child, history=None, key=identity):
        self.seen = dict()
        self.key = key
        if history:
            from zict import LRU
            self.seen = LRU(history, self.seen)

        Stream.__init__(self, child)
Beispiel #14
0
def test_mapping():
    """
    Test mapping interface for LRU().
    """
    d = {}
    # 100 is more than the max length when running check_mapping()
    lru = LRU(100, d)
    utils_test.check_mapping(lru)
    utils_test.check_closing(lru)
Beispiel #15
0
def test_callbacks():
    L = list()
    d = dict()
    lru = LRU(2, d, on_evict=lambda k, v: L.append((k, v)))

    lru['x'] = 1
    lru['y'] = 2
    lru['z'] = 3

    assert L == [('x', 1)]
Beispiel #16
0
def test_str():
    d = dict()
    lru = LRU(2, d)

    lru["x"] = 1
    lru["y"] = 2

    assert str(lru.total_weight) in str(lru)
    assert str(lru.total_weight) in repr(lru)
    assert str(lru.n) in str(lru)
    assert str(lru.n) in repr(lru)
    assert "dict" in str(lru)
    assert "dict" in repr(lru)
Beispiel #17
0
def test_str():
    d = dict()
    lru = LRU(2, d)

    lru['x'] = 1
    lru['y'] = 2

    assert str(lru.total_weight) in str(lru)
    assert str(lru.total_weight) in repr(lru)
    assert str(lru.n) in str(lru)
    assert str(lru.n) in repr(lru)
    assert 'dict' in str(lru)
    assert 'dict' in repr(lru)
Beispiel #18
0
def test_callbacks():
    count = [0]
    def cb(k, v):
        count[0] += 1

    L = list()
    d = dict()
    lru = LRU(2, d, on_evict=[lambda k, v: L.append((k, v)), cb])

    lru['x'] = 1
    lru['y'] = 2
    lru['z'] = 3

    assert L == [('x', 1)]
    assert count[0] == len(L)
Beispiel #19
0
def test_weight():
    d = dict()
    weight = lambda k, v: v
    lru = LRU(10, d, weight=weight)

    lru["x"] = 5
    assert lru.total_weight == 5

    lru["y"] = 4
    assert lru.total_weight == 9

    lru["z"] = 3
    assert d == {"y": 4, "z": 3}
    assert lru.total_weight == 7

    del lru["z"]
    assert lru.total_weight == 4

    lru["a"] = 10000
    assert "a" not in lru
    assert d == {"y": 4}
Beispiel #20
0
def test_weight():
    d = dict()
    weight = lambda k, v: v
    lru = LRU(10, d, weight=weight)

    lru['x'] = 5
    assert lru.total_weight == 5

    lru['y'] = 4
    assert lru.total_weight == 9

    lru['z'] = 3
    assert d == {'y': 4, 'z': 3}
    assert lru.total_weight == 7

    del lru['z']
    assert lru.total_weight == 4

    lru['a'] = 10000
    assert 'a' not in lru
    assert d == {'y': 4}
Beispiel #21
0
 def setup(self, worker):
     self.cache = Buffer(
         fast={},
         slow=Func(
             dump=blosc.pack_array,
             load=blosc.unpack_array,
             d=Buffer(
                 fast={},
                 slow=LRU(
                     n=self._maxdisk,
                     d=File(os.path.join(worker.local_directory, 'cache')),
                     weight=lambda k, v: len(v),
                 ),
                 n=self._maxcompressed,
                 weight=lambda k, v: len(v),
             ),
         ),
         n=self._maxmem,
         weight=lambda k, v: v.nbytes,
     )
     self.lock = Lock()
     self.hits = 0
     self.misses = 0
Beispiel #22
0
 def setup(self, worker):
     self.cache = LRU(n=self._maxmem, d={}, weight=lambda k, v: v.nbytes)
     self.lock = Lock()
     self.hits = 0
     self.misses = 0
Beispiel #23
0
class unique(Stream):
    """ Avoid sending through repeated elements

    This deduplicates a stream so that only new elements pass through.
    You can control how much of a history is stored with the ``maxsize=``
    parameter.  For example setting ``maxsize=1`` avoids sending through
    elements when one is repeated right after the other.

    Parameters
    ----------
    maxsize: int or None, optional
        number of stored unique values to check against
    key : function, optional
        Function which returns a representation of the incoming data.
        For example ``key=lambda x: x['a']`` could be used to allow only
        pieces of data with unique ``'a'`` values to pass through.
    hashable : bool, optional
        If True then data is assumed to be hashable, else it is not. This is
        used for determining how to cache the history, if hashable then
        either dicts or LRU caches are used, otherwise a deque is used.
        Defaults to True.

    Examples
    --------
    >>> source = Stream()
    >>> source.unique(maxsize=1).sink(print)
    >>> for x in [1, 1, 2, 2, 2, 1, 3]:
    ...     source.emit(x)
    1
    2
    1
    3
    """
    def __init__(self,
                 upstream,
                 maxsize=None,
                 key=identity,
                 hashable=True,
                 **kwargs):
        self.key = key
        self.maxsize = maxsize
        if hashable:
            self.seen = dict()
            if self.maxsize:
                from zict import LRU
                self.seen = LRU(self.maxsize, self.seen)
        else:
            self.seen = []

        Stream.__init__(self, upstream, **kwargs)

    def update(self, x, who=None):
        y = self.key(x)
        emit = True
        if isinstance(self.seen, list):
            if y in self.seen:
                self.seen.remove(y)
                emit = False
            self.seen.insert(0, y)
            if self.maxsize:
                del self.seen[self.maxsize:]
            if emit:
                return self._emit(x)
        else:
            if self.seen.get(y, '~~not_seen~~') == '~~not_seen~~':
                self.seen[y] = 1
                return self._emit(x)