def test_get_cache_items(tmpdir):
"""Test cache items listing."""
def func(arg):
"""Dummy function."""
return arg
register_hdfs_store_backend()
mem = Memory(location=tmpdir.strpath,
host=__namenode__,
backend='hdfs',
user='******',
verbose=100,
compress=False)
assert not mem.store.get_cache_items()
cached_func = mem.cache(func)
for arg in ["test1", "test2", "test3"]:
cached_func(arg)
# get_cache_items always returns an empty list for the moment
assert len(mem.store.get_cache_items()) == 3
mem.clear()
assert not mem.store.get_cache_items()
def test_root_location_replacement(tmpdir):
"""Test that root location is correctly replaced."""
location = tmpdir.strpath
register_hdfs_store_backend()
mem = Memory(location=location,
host=__namenode__,
backend='hdfs',
user='******',
verbose=100)
assert mem.store.cachedir == os.path.join(tmpdir.strpath[1:], "joblib")
def test_store_and_retrieve(capsys, tmpdir, compress, arg):
"""Test that any types can be cached in hdfs store."""
def func(arg):
"""Dummy function."""
print("executing function")
return arg
register_hdfs_store_backend()
mem = Memory(location=tmpdir.strpath[1:],
host=__namenode__,
backend='hdfs',
user='******',
verbose=0,
compress=compress)
assert mem.store.cachedir == os.path.join(tmpdir.strpath[1:], "joblib")
func = mem.cache(func)
# First call executes and persists result
result = func(arg)
if isinstance(arg, np.ndarray):
np.testing.assert_array_equal(arg, result)
else:
assert result == arg
out, err = capsys.readouterr()
assert out == "executing function\n"
assert not err
# Second call returns the cached result
result = func(arg)
if isinstance(arg, np.ndarray):
np.testing.assert_array_equal(arg, result)
else:
assert result == arg
out, err = capsys.readouterr()
assert not out
assert not err
def test_passing_backend_base_to_memory(tmpdir):
"""Test passing a store as location in memory is correctly handled."""
register_hdfs_store_backend()
mem = Memory(location=tmpdir.strpath,
host=__namenode__,
backend='hdfs',
user='******',
verbose=100)
assert mem.store.cachedir == os.path.join(tmpdir.strpath[1:], "joblib")
mem2 = Memory(location=mem.store,
host=__namenode__,
backend='hdfs',
user='******',
verbose=100)
assert mem2.store.cachedir == mem.store.cachedir
def test_clear_cache(tmpdir):
"""Test clearing cache."""
def func(arg):
"""Dummy function."""
print("executing function")
return arg
register_hdfs_store_backend()
mem = Memory(location=tmpdir.strpath,
host=__namenode__,
backend='hdfs',
user='******',
verbose=100,
compress=False)
cached_func = mem.cache(func)
cached_func("test")
mem.clear()
assert not mem.store.object_exists(mem.store.cachedir)
"""Example showing how to use joblib-hadoop with an HDFS store"""
import numpy as np
from joblib import Memory
from joblibhadoop.hdfs import register_hdfs_store_backend
if __name__ == '__main__':
register_hdfs_store_backend()
mem = Memory(location='joblib_cache_hdfs',
backend='hdfs', host='namenode', port=9000, user='******',
verbose=100, compress=True)
mem.clear()
multiply = mem.cache(np.multiply)
array1 = np.arange(1000)
array2 = np.arange(1000)
print("# First call")
_ = multiply(array1, array2)
print("# Second call")
# Second call should return the cached result
result = multiply(array1, array2)
print(result)
def test_hdfs_store_backend_registration():
"""Smoke test for hdfs backend registration."""
register_hdfs_store_backend()
assert "hdfs" in _STORE_BACKENDS
assert _STORE_BACKENDS["hdfs"] == HDFSStoreBackend