def test_get_cache_items(tmpdir): """Test cache items listing.""" def func(arg): """Dummy function.""" return arg register_hdfs_store_backend() mem = Memory(location=tmpdir.strpath, host=__namenode__, backend='hdfs', user='******', verbose=100, compress=False) assert not mem.store.get_cache_items() cached_func = mem.cache(func) for arg in ["test1", "test2", "test3"]: cached_func(arg) # get_cache_items always returns an empty list for the moment assert len(mem.store.get_cache_items()) == 3 mem.clear() assert not mem.store.get_cache_items()
def test_root_location_replacement(tmpdir): """Test that root location is correctly replaced.""" location = tmpdir.strpath register_hdfs_store_backend() mem = Memory(location=location, host=__namenode__, backend='hdfs', user='******', verbose=100) assert mem.store.cachedir == os.path.join(tmpdir.strpath[1:], "joblib")
def test_store_and_retrieve(capsys, tmpdir, compress, arg): """Test that any types can be cached in hdfs store.""" def func(arg): """Dummy function.""" print("executing function") return arg register_hdfs_store_backend() mem = Memory(location=tmpdir.strpath[1:], host=__namenode__, backend='hdfs', user='******', verbose=0, compress=compress) assert mem.store.cachedir == os.path.join(tmpdir.strpath[1:], "joblib") func = mem.cache(func) # First call executes and persists result result = func(arg) if isinstance(arg, np.ndarray): np.testing.assert_array_equal(arg, result) else: assert result == arg out, err = capsys.readouterr() assert out == "executing function\n" assert not err # Second call returns the cached result result = func(arg) if isinstance(arg, np.ndarray): np.testing.assert_array_equal(arg, result) else: assert result == arg out, err = capsys.readouterr() assert not out assert not err
def test_passing_backend_base_to_memory(tmpdir): """Test passing a store as location in memory is correctly handled.""" register_hdfs_store_backend() mem = Memory(location=tmpdir.strpath, host=__namenode__, backend='hdfs', user='******', verbose=100) assert mem.store.cachedir == os.path.join(tmpdir.strpath[1:], "joblib") mem2 = Memory(location=mem.store, host=__namenode__, backend='hdfs', user='******', verbose=100) assert mem2.store.cachedir == mem.store.cachedir
def test_clear_cache(tmpdir): """Test clearing cache.""" def func(arg): """Dummy function.""" print("executing function") return arg register_hdfs_store_backend() mem = Memory(location=tmpdir.strpath, host=__namenode__, backend='hdfs', user='******', verbose=100, compress=False) cached_func = mem.cache(func) cached_func("test") mem.clear() assert not mem.store.object_exists(mem.store.cachedir)
"""Example showing how to use joblib-hadoop with an HDFS store""" import numpy as np from joblib import Memory from joblibhadoop.hdfs import register_hdfs_store_backend if __name__ == '__main__': register_hdfs_store_backend() mem = Memory(location='joblib_cache_hdfs', backend='hdfs', host='namenode', port=9000, user='******', verbose=100, compress=True) mem.clear() multiply = mem.cache(np.multiply) array1 = np.arange(1000) array2 = np.arange(1000) print("# First call") _ = multiply(array1, array2) print("# Second call") # Second call should return the cached result result = multiply(array1, array2) print(result)
def test_hdfs_store_backend_registration(): """Smoke test for hdfs backend registration.""" register_hdfs_store_backend() assert "hdfs" in _STORE_BACKENDS assert _STORE_BACKENDS["hdfs"] == HDFSStoreBackend