def _posttask(self, key, value, dsk, state, id):
duration = default_timer() - self.starttimes[key]
deps = state['dependencies'][key]
if deps:
duration += max(self.durations.get(k, 0) for k in deps)
self.durations[key] = duration
nb = cachey.nbytes(value) + overhead + sys.getsizeof(key) * 4
self.cache.put(key, value, cost=duration / nb / 1e9, nbytes=nb)
def _posttask(self, key, value, dsk, state, id):
duration = default_timer() - self.starttimes[key]
deps = state['dependencies'][key]
if deps:
duration += max(self.durations.get(k, 0) for k in deps)
self.durations[key] = duration
nb = cachey.nbytes(value) + overhead + sys.getsizeof(key) * 4
self.cache.put(key, value, cost=duration / nb / 1e9, nbytes=nb)
def test_cache_data_dict():
my_dict = {}
c = Cache(available_bytes=nbytes(1) * 3, cache_data=my_dict)
c.put('x', 1, 10)
assert c.get('x') == 1
assert my_dict['x'] == 1
c.clear()
assert 'x' not in c
def test_pandas():
x = np.random.random(1000)
i = np.random.random(1000)
s = pd.Series(x, index=i)
assert nbytes(s) == nbytes(x) + nbytes(i)
df = pd.DataFrame(s)
assert nbytes(df) == nbytes(s)
s = pd.Series(pd.Categorical(['a', 'b'] * 1000))
assert nbytes(s.cat.codes) < nbytes(s) < nbytes(s.cat.codes) * 2
def test_pandas():
x = np.random.random(1000)
i = np.random.random(1000)
s = pd.Series(x, index=i)
assert nbytes(s) == nbytes(x) + nbytes(i)
df = pd.DataFrame(s)
assert nbytes(df) == nbytes(s)
s = pd.Series(pd.Categorical(['a', 'b'] * 1000))
assert nbytes(s.cat.codes) < nbytes(s) < nbytes(s.cat.codes) * 2
def test_cache_scores_update():
c = Cache(available_bytes=nbytes(1) * 2)
c.put('x', 1, 1)
c.put('y', 1, 1)
c.get('x')
c.get('x')
c.get('x')
c.put('z', 1, 1)
assert set(c.data) == set('xz')
def test_cache_scores_update():
c = Cache(available_bytes=nbytes(1) * 2)
c.put('x', 1, 1)
c.put('y', 1, 1)
c.get('x')
c.get('x')
c.get('x')
c.put('z', 1, 1)
assert set(c.data) == set('xz')
def test_cache_resize():
c = Cache(available_bytes=nbytes(1) * 3)
c.put('x', 1, 10)
assert c.get('x') == 1
assert 'x' in c
c.put('a', 1, 10)
c.put('b', 1, 10)
c.put('c', 1, 10)
assert set(c.data) == set('xbc')
c.put('d', 1, 10)
assert set(c.data) == set('xcd')
# resize will shrink
c.resize(available_bytes=nbytes(1) * 1)
assert set(c.data) == set('x')
c.resize(available_bytes=nbytes(1) * 10)
assert set(c.data) == set('x')
def test_memoize():
c = Cache(available_bytes=nbytes(1) * 3)
flag = [0]
def slow_inc(x):
flag[0] += 1
sleep(0.01)
return x + 1
memo_inc = c.memoize(slow_inc)
assert memo_inc(1) == 2
assert memo_inc(1) == 2
assert list(c.data.values()) == [2]
def test_memoize():
c = Cache(available_bytes=nbytes(1) * 3)
flag = [0]
def slow_inc(x):
flag[0] += 1
sleep(0.01)
return x + 1
memo_inc = c.memoize(slow_inc)
assert memo_inc(1) == 2
assert memo_inc(1) == 2
assert list(c.data.values()) == [2]
def test_cache():
c = Cache(available_bytes=nbytes(1) * 3)
c.put('x', 1, 10)
assert c.get('x') == 1
assert 'x' in c
c.put('a', 1, 10)
c.put('b', 1, 10)
c.put('c', 1, 10)
assert set(c.data) == set('xbc')
c.put('d', 1, 10)
assert set(c.data) == set('xcd')
c.clear()
assert 'x' not in c
assert not c.data
assert not c.heap
def test_cache():
c = Cache(available_bytes=nbytes(1) * 3)
c.put('x', 1, 10)
assert c.get('x') == 1
assert 'x' in c
c.put('a', 1, 10)
c.put('b', 1, 10)
c.put('c', 1, 10)
assert set(c.data) == set('xbc')
c.put('d', 1, 10)
assert set(c.data) == set('xcd')
c.clear()
assert 'x' not in c
assert not c.data
assert not c.heap
def test_obj():
assert nbytes('hello'*100) > 500
def test_obj():
assert nbytes('hello' * 100) > 500
