def test_cache_scores_update():
c = Cache(available_bytes=nbytes(1) * 2)
c.put('x', 1, 1)
c.put('y', 1, 1)
c.get('x')
c.get('x')
c.get('x')
c.put('z', 1, 1)
assert set(c.data) == set('xz')
def test_just_one_reference():
c = Cache(available_bytes=1000)
o = object()
x = sys.getrefcount(o)
c.put('key', o, cost=10)
y = sys.getrefcount(o)
assert y == x + 1
c.retire('key')
z = sys.getrefcount(o)
assert z == x
def test_cache_scores_update():
c = Cache(available_bytes=nbytes(1) * 2)
c.put('x', 1, 1)
c.put('y', 1, 1)
c.get('x')
c.get('x')
c.get('x')
c.put('z', 1, 1)
assert set(c.data) == set('xz')
def get_zvariables(dataset: xr.Dataset = Depends(get_dataset),
cache: cachey.Cache = Depends(get_cache)):
"""FastAPI dependency that returns a dictionary of zarr encoded variables."""
cache_key = dataset.attrs.get(DATASET_ID_ATTR_KEY, '') + '/' + 'zvariables'
zvariables = cache.get(cache_key)
if zvariables is None:
zvariables = create_zvariables(dataset)
# we want to permanently cache this: set high cost value
cache.put(cache_key, zvariables, 99999)
return zvariables
def get_variable_chunk(
var: str,
chunk: str,
dataset: xr.Dataset = Depends(get_dataset),
cache: cachey.Cache = Depends(get_cache),
zvariables: dict = Depends(get_zvariables),
zmetadata: dict = Depends(_get_zmetadata),
):
"""Get a zarr array chunk.
This will return cached responses when available.
"""
# First check that this request wasn't for variable metadata
if array_meta_key in chunk:
return zmetadata['metadata'][f'{var}/{array_meta_key}']
elif attrs_key in chunk:
return zmetadata['metadata'][f'{var}/{attrs_key}']
elif group_meta_key in chunk:
raise HTTPException(status_code=404, detail='No subgroups')
else:
logger.debug('var is %s', var)
logger.debug('chunk is %s', chunk)
cache_key = dataset.attrs.get(DATASET_ID_ATTR_KEY,
'') + '/' + f'{var}/{chunk}'
response = cache.get(cache_key)
if response is None:
with CostTimer() as ct:
arr_meta = zmetadata['metadata'][f'{var}/{array_meta_key}']
da = zvariables[var].data
data_chunk = get_data_chunk(da,
chunk,
out_shape=arr_meta['chunks'])
echunk = encode_chunk(
data_chunk.tobytes(),
filters=arr_meta['filters'],
compressor=arr_meta['compressor'],
)
response = Response(echunk,
media_type='application/octet-stream')
cache.put(cache_key, response, ct.time, len(echunk))
return response
def test_memoize():
c = Cache(available_bytes=nbytes(1) * 3)
flag = [0]
def slow_inc(x):
flag[0] += 1
sleep(0.01)
return x + 1
memo_inc = c.memoize(slow_inc)
assert memo_inc(1) == 2
assert memo_inc(1) == 2
assert list(c.data.values()) == [2]
def get_zmetadata(
dataset: xr.Dataset = Depends(get_dataset),
cache: cachey.Cache = Depends(get_cache),
zvariables: dict = Depends(get_zvariables),
):
"""FastAPI dependency that returns a consolidated zmetadata dictionary. """
zmeta = cache.get(zarr_metadata_key)
if zmeta is None:
zmeta = create_zmetadata(dataset)
# we want to permanently cache this: set high cost value
cache.put(zarr_metadata_key, zmeta, 99999)
return zmeta
def test_memoize():
c = Cache(available_bytes=nbytes(1) * 3)
flag = [0]
def slow_inc(x):
flag[0] += 1
sleep(0.01)
return x + 1
memo_inc = c.memoize(slow_inc)
assert memo_inc(1) == 2
assert memo_inc(1) == 2
assert list(c.data.values()) == [2]
def test_cache_data_dict():
my_dict = {}
c = Cache(available_bytes=nbytes(1) * 3, cache_data=my_dict)
c.put('x', 1, 10)
assert c.get('x') == 1
assert my_dict['x'] == 1
c.clear()
assert 'x' not in c
def test_just_one_reference():
c = Cache(available_bytes=1000)
o = object()
x = sys.getrefcount(o)
c.put('key', o, cost=10)
y = sys.getrefcount(o)
assert y == x + 1
c.retire('key')
z = sys.getrefcount(o)
assert z == x
def test_callbacks():
hit_flag = [False]
def hit(key, value):
hit_flag[0] = (key, value)
miss_flag = [False]
def miss(key):
miss_flag[0] = key
c = Cache(100, hit=hit, miss=miss)
c.get('x')
assert miss_flag[0] == 'x'
assert hit_flag[0] == False
c.put('y', 1, 1)
c.get('y')
assert hit_flag[0] == ('y', 1)
def test_callbacks():
hit_flag = [False]
def hit(key, value):
hit_flag[0] = (key, value)
miss_flag = [False]
def miss(key):
miss_flag[0] = key
c = Cache(100, hit=hit, miss=miss)
c.get('x')
assert miss_flag[0] == 'x'
assert hit_flag[0] == False
c.put('y', 1, 1)
c.get('y')
assert hit_flag[0] == ('y', 1)
def test_cache():
c = Cache(available_bytes=nbytes(1) * 3)
c.put('x', 1, 10)
assert c.get('x') == 1
assert 'x' in c
c.put('a', 1, 10)
c.put('b', 1, 10)
c.put('c', 1, 10)
assert set(c.data) == set('xbc')
c.put('d', 1, 10)
assert set(c.data) == set('xcd')
c.clear()
assert 'x' not in c
assert not c.data
assert not c.heap
def test_cache_resize():
c = Cache(available_bytes=nbytes(1) * 3)
c.put('x', 1, 10)
assert c.get('x') == 1
assert 'x' in c
c.put('a', 1, 10)
c.put('b', 1, 10)
c.put('c', 1, 10)
assert set(c.data) == set('xbc')
c.put('d', 1, 10)
assert set(c.data) == set('xcd')
# resize will shrink
c.resize(available_bytes=nbytes(1) * 1)
assert set(c.data) == set('x')
c.resize(available_bytes=nbytes(1) * 10)
assert set(c.data) == set('x')
ax.set_xticks(np.arange(0, 24, 1))
ax.set_xlabel('Start Time (hours)')
ax.set_ylabel('Travel Time (seconds)')
ax.set_title('Travel Time During the Day')
plt.plot(x, y)
plt.grid()
plt.savefig("mygraph.png")
def predict(model, row, time_of_day=None, buffer=1000, buffer_samples=21):
rows = []
time_of_day = time_of_day or row['time_of_day']
for i in np.linspace(max(time_of_day - buffer, 0), min(time_of_day + buffer, 60 * 60 * 24), buffer_samples):
row = row.copy()
row['time_of_day'] = i
rows.append(row)
preds = model.predict(pd.DataFrame(rows))
return preds.mean()
def model_predict(model, row, time_of_day=None):
row['time_of_day'] = time_of_day or row.get('time_of_day')
pred = model.predict(pd.DataFrame([row]))
return pred[0]
c = Cache(1e9)
def get_regressor(model, x, predict):
regressor = lambda t: predict(model, x, t)
regressor = c.memoize(regressor) # In order to speed up the evaluation - Cache is used on the regressor
return regressor
def cache(self):
""" Cache Property """
if self._cache is None:
self._cache = Cache(**self._cache_kws)
return self._cache
def test_cache():
c = Cache(available_bytes=nbytes(1) * 3)
c.put('x', 1, 10)
assert c.get('x') == 1
assert 'x' in c
c.put('a', 1, 10)
c.put('b', 1, 10)
c.put('c', 1, 10)
assert set(c.data) == set('xbc')
c.put('d', 1, 10)
assert set(c.data) == set('xcd')
c.clear()
assert 'x' not in c
assert not c.data
assert not c.heap