Beispiel #1
0
 def _posttask(self, key, value, dsk, state, id):
     duration = default_timer() - self.starttimes[key]
     deps = state['dependencies'][key]
     if deps:
         duration += max(self.durations.get(k, 0) for k in deps)
     self.durations[key] = duration
     nb = cachey.nbytes(value) + overhead + sys.getsizeof(key) * 4
     self.cache.put(key, value, cost=duration / nb / 1e9, nbytes=nb)
Beispiel #2
0
 def _posttask(self, key, value, dsk, state, id):
     duration = default_timer() - self.starttimes[key]
     deps = state['dependencies'][key]
     if deps:
         duration += max(self.durations.get(k, 0) for k in deps)
     self.durations[key] = duration
     nb = cachey.nbytes(value) + overhead + sys.getsizeof(key) * 4
     self.cache.put(key, value, cost=duration / nb / 1e9, nbytes=nb)
Beispiel #3
0
def test_cache_data_dict():

    my_dict = {}
    c = Cache(available_bytes=nbytes(1) * 3, cache_data=my_dict)
    c.put('x', 1, 10)
    assert c.get('x') == 1
    assert my_dict['x'] == 1
    c.clear()
    assert 'x' not in c
Beispiel #4
0
    def test_pandas():
        x = np.random.random(1000)
        i = np.random.random(1000)
        s = pd.Series(x, index=i)
        assert nbytes(s) == nbytes(x) + nbytes(i)

        df = pd.DataFrame(s)
        assert nbytes(df) == nbytes(s)

        s = pd.Series(pd.Categorical(['a', 'b'] * 1000))
        assert nbytes(s.cat.codes) < nbytes(s) < nbytes(s.cat.codes) * 2
Beispiel #5
0
    def test_pandas():
        x = np.random.random(1000)
        i = np.random.random(1000)
        s = pd.Series(x, index=i)
        assert nbytes(s) == nbytes(x) + nbytes(i)

        df = pd.DataFrame(s)
        assert nbytes(df) == nbytes(s)

        s = pd.Series(pd.Categorical(['a', 'b'] * 1000))
        assert nbytes(s.cat.codes) < nbytes(s) < nbytes(s.cat.codes) * 2
Beispiel #6
0
def test_cache_scores_update():
    c = Cache(available_bytes=nbytes(1) * 2)
    c.put('x', 1, 1)
    c.put('y', 1, 1)
    c.get('x')
    c.get('x')
    c.get('x')

    c.put('z', 1, 1)
    assert set(c.data) == set('xz')
Beispiel #7
0
def test_cache_scores_update():
    c = Cache(available_bytes=nbytes(1) * 2)
    c.put('x', 1, 1)
    c.put('y', 1, 1)
    c.get('x')
    c.get('x')
    c.get('x')

    c.put('z', 1, 1)
    assert set(c.data) == set('xz')
Beispiel #8
0
def test_cache_resize():
    c = Cache(available_bytes=nbytes(1) * 3)

    c.put('x', 1, 10)
    assert c.get('x') == 1
    assert 'x' in c

    c.put('a', 1, 10)
    c.put('b', 1, 10)
    c.put('c', 1, 10)
    assert set(c.data) == set('xbc')
    c.put('d', 1, 10)
    assert set(c.data) == set('xcd')

    # resize will shrink
    c.resize(available_bytes=nbytes(1) * 1)

    assert set(c.data) == set('x')

    c.resize(available_bytes=nbytes(1) * 10)

    assert set(c.data) == set('x')
Beispiel #9
0
def test_memoize():
    c = Cache(available_bytes=nbytes(1) * 3)

    flag = [0]
    def slow_inc(x):
        flag[0] += 1
        sleep(0.01)
        return x + 1

    memo_inc = c.memoize(slow_inc)

    assert memo_inc(1) == 2
    assert memo_inc(1) == 2

    assert list(c.data.values()) == [2]
Beispiel #10
0
def test_memoize():
    c = Cache(available_bytes=nbytes(1) * 3)

    flag = [0]

    def slow_inc(x):
        flag[0] += 1
        sleep(0.01)
        return x + 1

    memo_inc = c.memoize(slow_inc)

    assert memo_inc(1) == 2
    assert memo_inc(1) == 2

    assert list(c.data.values()) == [2]
Beispiel #11
0
def test_cache():
    c = Cache(available_bytes=nbytes(1) * 3)

    c.put('x', 1, 10)
    assert c.get('x') == 1
    assert 'x' in c

    c.put('a', 1, 10)
    c.put('b', 1, 10)
    c.put('c', 1, 10)
    assert set(c.data) == set('xbc')
    c.put('d', 1, 10)
    assert set(c.data) == set('xcd')

    c.clear()
    assert 'x' not in c
    assert not c.data
    assert not c.heap
Beispiel #12
0
def test_cache():
    c = Cache(available_bytes=nbytes(1) * 3)

    c.put('x', 1, 10)
    assert c.get('x') == 1
    assert 'x' in c

    c.put('a', 1, 10)
    c.put('b', 1, 10)
    c.put('c', 1, 10)
    assert set(c.data) == set('xbc')
    c.put('d', 1, 10)
    assert set(c.data) == set('xcd')

    c.clear()
    assert 'x' not in c
    assert not c.data
    assert not c.heap
Beispiel #13
0
def test_obj():
    assert nbytes('hello'*100) > 500
Beispiel #14
0
def test_obj():
    assert nbytes('hello' * 100) > 500