def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
memory.clear()
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
assert g._get_func_dir() == path
assert os.path.exists(path)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
dir, _ = g.get_output_dir(1)
a = g(1)
assert os.path.exists(dir)
os.remove(os.path.join(dir, 'output.pkl'))
assert a == g(1)
def test_memory_func_with_kwonly_args():
mem = Memory(cachedir=env['dir'], verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
nose.tools.assert_equal(func_cached(1, 2, kw1=3), (1, 2, 3, 'kw2'))
# Making sure that providing a keyword-only argument by
# position raises an exception
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached,
1, 2, 3, {'kw2': 4})
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached,
1, 2, 3, {'kw2': 4})
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2=4), (1, 2, 3, 4))
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2='ignored'), (1, 2, 3, 4))
def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(location=tmpdir.strpath, verbose=0)
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
func_id = _build_func_identifier(f)
location = os.path.join(g.store_backend.location, func_id)
assert location == path
assert os.path.exists(path)
assert memory.location == os.path.dirname(g.store_backend.location)
with warns(DeprecationWarning) as w:
assert memory.cachedir == g.store_backend.location
assert len(w) == 1
assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
func_id, args_id = g._get_output_identifiers(1)
output_dir = os.path.join(g.store_backend.location, func_id, args_id)
a = g(1)
assert os.path.exists(output_dir)
os.remove(os.path.join(output_dir, 'output.pkl'))
assert a == g(1)
def test_func_dir():
# Test the creation of the memory cache directory for the function.
memory = Memory(cachedir=env["dir"], verbose=0)
memory.clear()
path = __name__.split(".")
path.append("f")
path = os.path.join(env["dir"], "joblib", *path)
g = memory.cache(f)
# Test that the function directory is created on demand
yield nose.tools.assert_equal, g._get_func_dir(), path
yield nose.tools.assert_true, os.path.exists(path)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
yield nose.tools.assert_false, g._check_previous_func_code()
yield nose.tools.assert_true, os.path.exists(os.path.join(path, "func_code.py"))
yield nose.tools.assert_true, g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
dir, _ = g.get_output_dir(1)
a = g(1)
yield nose.tools.assert_true, os.path.exists(dir)
os.remove(os.path.join(dir, "output.pkl"))
yield nose.tools.assert_equal, a, g(1)
def test_call_and_shelve_lazily_load_stored_result(tmpdir):
"""Check call_and_shelve only load stored data if needed."""
test_access_time_file = tmpdir.join('test_access')
test_access_time_file.write('test_access')
test_access_time = os.stat(test_access_time_file.strpath).st_atime
# check file system access time stats resolution is lower than test wait
# timings.
time.sleep(0.5)
assert test_access_time_file.read() == 'test_access'
if test_access_time == os.stat(test_access_time_file.strpath).st_atime:
# Skip this test when access time cannot be retrieved with enough
# precision from the file system (e.g. NTFS on windows).
pytest.skip("filesystem does not support fine-grained access time "
"attribute")
memory = Memory(location=tmpdir.strpath, verbose=0)
func = memory.cache(f)
func_id, argument_hash = func._get_output_identifiers(2)
result_path = os.path.join(memory.store_backend.location,
func_id, argument_hash, 'output.pkl')
assert func(2) == 5
first_access_time = os.stat(result_path).st_atime
time.sleep(1)
# Should not access the stored data
result = func.call_and_shelve(2)
assert isinstance(result, MemorizedResult)
assert os.stat(result_path).st_atime == first_access_time
time.sleep(1)
# Read the stored data => last access time is greater than first_access
assert result.get() == 5
assert os.stat(result_path).st_atime > first_access_time
def test_memory_warning_collision_detection():
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(cachedir=env['dir'], verbose=0)
# For isolation with other tests
memory.clear()
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
if not hasattr(warnings, 'catch_warnings'):
# catch_warnings is new in Python 2.6
return
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# This is a temporary workaround until we get rid of
# inspect.getargspec, see
# https://github.com/joblib/joblib/issues/247
warnings.simplefilter("ignore", DeprecationWarning)
a1(1)
b1(1)
a1(0)
yield nose.tools.assert_equal, len(w), 2
yield nose.tools.assert_true, \
"cannot detect" in str(w[-1].message).lower()
def test_memory_pickle_dump_load(tmpdir, memory_kwargs):
memory = Memory(location=tmpdir.strpath, **memory_kwargs)
memory_reloaded = pickle.loads(pickle.dumps(memory))
# Compare Memory instance before and after pickle roundtrip
compare(memory.store_backend, memory_reloaded.store_backend)
compare(memory, memory_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(memory) == hash(memory_reloaded)
func_cached = memory.cache(f)
func_cached_reloaded = pickle.loads(pickle.dumps(func_cached))
# Compare MemorizedFunc instance before/after pickle roundtrip
compare(func_cached.store_backend, func_cached_reloaded.store_backend)
compare(func_cached, func_cached_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(func_cached) == hash(func_cached_reloaded)
# Compare MemorizedResult instance before/after pickle roundtrip
memorized_result = func_cached.call_and_shelve(1)
memorized_result_reloaded = pickle.loads(pickle.dumps(memorized_result))
compare(memorized_result.store_backend,
memorized_result_reloaded.store_backend)
compare(memorized_result, memorized_result_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(memorized_result) == hash(memorized_result_reloaded)
def test_memory_objects_repr(tmpdir):
# Verify printable reprs of MemorizedResult, MemorizedFunc and Memory.
def my_func(a, b):
return a + b
memory = Memory(location=tmpdir.strpath, verbose=0)
memorized_func = memory.cache(my_func)
memorized_func_repr = 'MemorizedFunc(func={func}, location={location})'
assert str(memorized_func) == memorized_func_repr.format(
func=my_func,
location=memory.store_backend.location)
memorized_result = memorized_func.call_and_shelve(42, 42)
memorized_result_repr = ('MemorizedResult(location="{location}", '
'func="{func}", args_id="{args_id}")')
assert str(memorized_result) == memorized_result_repr.format(
location=memory.store_backend.location,
func=memorized_result.func_id,
args_id=memorized_result.args_id)
assert str(memory) == 'Memory(location={location})'.format(
location=memory.store_backend.location)
def test_memory_func_with_kwonly_args(tmpdir):
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
assert func_cached(1, 2, kw1=3) == (1, 2, 3, 'kw2')
# Making sure that providing a keyword-only argument by
# position raises an exception
with raises(ValueError) as excinfo:
func_cached(1, 2, 3, kw2=4)
excinfo.match("Keyword-only parameter 'kw1' was passed as positional "
"parameter")
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
with raises(ValueError) as excinfo:
func_cached(1, 2, 3, kw2=4)
excinfo.match("Keyword-only parameter 'kw1' was passed as positional "
"parameter")
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
assert func_cached(1, 2, kw1=3, kw2=4) == (1, 2, 3, 4)
assert func_cached(1, 2, kw1=3, kw2='ignored') == (1, 2, 3, 4)
def test_memory_arg_nested():
" Test memory with a nested function argument."
memory = Memory(cachedir=env['dir'], verbose=0)
memory.clear(warn=False)
accum = {'value': 0}
def func_1():
return 1
def func_2():
return 2
def decorator(func_):
def decorated():
return func_()
return decorated
@memory.cache()
def run_func(func):
accum['value'] += 1
return func()
a = run_func(decorator(func_1))
b = run_func(decorator(func_1))
c = run_func(decorator(func_2))
nose.tools.assert_equal(a, 1)
nose.tools.assert_equal(b, 1)
nose.tools.assert_equal(c, 2)
nose.tools.assert_equal(accum['value'], 2)
def test_memory_eval(tmpdir):
" Smoke test memory with a function with a function defined in an eval."
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
m = eval('lambda x: x')
mm = memory.cache(m)
assert mm(1) == 1
def test_memory_eval():
" Smoke test memory with a function with a function defined in an eval."
memory = Memory(cachedir=env["dir"], verbose=0)
m = eval("lambda x: x")
mm = memory.cache(m)
yield nose.tools.assert_equal, 1, mm(1)
def test_call_and_shelve_argument_hash(tmpdir):
# Verify that a warning is raised when accessing arguments_hash
# attribute from MemorizedResult
func = Memory(location=tmpdir.strpath, verbose=0).cache(f)
result = func.call_and_shelve(2)
assert isinstance(result, MemorizedResult)
with warns(DeprecationWarning) as w:
assert result.argument_hash == result.args_id
assert len(w) == 1
assert "The 'argument_hash' attribute has been deprecated" \
in str(w[-1].message)
def test_memory_file_modification(capsys, tmpdir):
# Test that modifying a Python file after loading it does not lead to
# Recomputation
dir_name = tmpdir.mkdir('tmp_import').strpath
filename = os.path.join(dir_name, 'tmp_joblib_.py')
content = 'def f(x):\n print(x)\n return x\n'
with open(filename, 'w') as module_file:
module_file.write(content)
# Load the module:
sys.path.append(dir_name)
import tmp_joblib_ as tmp
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
f = mem.cache(tmp.f)
# First call f a few times
f(1)
f(2)
f(1)
# Now modify the module where f is stored without modifying f
with open(filename, 'w') as module_file:
module_file.write('\n\n' + content)
# And call f a couple more times
f(1)
f(1)
# Flush the .pyc files
shutil.rmtree(dir_name)
os.mkdir(dir_name)
# Now modify the module where f is stored, modifying f
content = 'def f(x):\n print("x=%s" % x)\n return x\n'
with open(filename, 'w') as module_file:
module_file.write(content)
# And call f more times prior to reloading: the cache should not be
# invalidated at this point as the active function definition has not
# changed in memory yet.
f(1)
f(1)
# Now reload
sys.stdout.write('Reloading\n')
sys.modules.pop('tmp_joblib_')
import tmp_joblib_ as tmp
f = mem.cache(tmp.f)
# And call f more times
f(1)
f(1)
out, err = capsys.readouterr()
assert out == '1\n2\nReloading\nx=1\n'
def test_argument_change(tmpdir):
""" Check that if a function has a side effect in its arguments, it
should use the hash of changing arguments.
"""
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
func = mem.cache(count_and_append)
# call the function for the first time, is should cache it with
# argument x=[]
assert func() == 0
# the second time the argument is x=[None], which is not cached
# yet, so the functions should be called a second time
assert func() == 1
def check_identity_lazy(func, accumulator, location):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(location=location, verbose=0)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
assert func(i) == i
assert len(accumulator) == i + 1
def test_memory_kwarg(tmpdir):
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
check_identity_lazy(g, accumulator, tmpdir.strpath)
memory = Memory(location=tmpdir.strpath, verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
assert g(l=30, m=2) == 30
def test_no_memory():
""" Test memory with cachedir=None: no memoize """
accumulator = list()
def ff(l):
accumulator.append(1)
return l
mem = Memory(cachedir=None, verbose=0)
gg = mem.cache(ff)
for _ in range(4):
current_accumulator = len(accumulator)
gg(1)
assert len(accumulator) == current_accumulator + 1
def test_no_memory():
""" Test memory with location=None: no memoize """
accumulator = list()
def ff(l):
accumulator.append(1)
return l
memory = Memory(location=None, verbose=0)
gg = memory.cache(ff)
for _ in range(4):
current_accumulator = len(accumulator)
gg(1)
assert len(accumulator) == current_accumulator + 1
def check_identity_lazy(func, accumulator):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(cachedir=env['dir'], verbose=0)
memory.clear(warn=False)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
yield nose.tools.assert_equal, func(i), i
yield nose.tools.assert_equal, len(accumulator), i + 1
def check_identity_lazy(func, accumulator):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(cachedir=env["dir"], verbose=0)
memory.clear(warn=False)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
yield nose.tools.assert_equal, func(i), i
yield nose.tools.assert_equal, len(accumulator), i + 1
def test_memory_kwarg(tmpdir):
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
check_identity_lazy(g, accumulator, tmpdir.strpath)
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
assert g(l=30, m=2) == 30
def test_memory_in_memory_function_code_change(tmpdir):
_function_to_cache.__code__ = _sum.__code__
memory = Memory(location=tmpdir.strpath, verbose=0)
f = memory.cache(_function_to_cache)
assert f(1, 2) == 3
assert f(1, 2) == 3
with warns(JobLibCollisionWarning):
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
assert f(1, 2) == 2
assert f(1, 2) == 2
def test_memory_in_memory_function_code_change(tmpdir):
_function_to_cache.__code__ = _sum.__code__
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
f = mem.cache(_function_to_cache)
assert f(1, 2) == 3
assert f(1, 2) == 3
with warns(JobLibCollisionWarning):
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
assert f(1, 2) == 2
assert f(1, 2) == 2
def test_no_memory():
""" Test memory with cachedir=None: no memoize """
accumulator = list()
def ff(l):
accumulator.append(1)
return l
mem = Memory(cachedir=None, verbose=0)
gg = mem.cache(ff)
for _ in range(4):
current_accumulator = len(accumulator)
gg(1)
yield nose.tools.assert_equal, len(accumulator), \
current_accumulator + 1
def test_cached(tmpdir):
" Test cached function."
def g(a, b, c, d, e=None, f=1):
return (a, b, c, d, e, f)
memory = Memory(location=tmpdir.strpath, verbose=0)
g = memory.cache(g)
args = (1, 2, 3, 4)
kwargs = {'e': 5, 'f': 6}
g(*args, **kwargs)
assert g.cached(*args, **kwargs)
assert not g.cached(*args[::-1], **kwargs)
g.clear()
assert not g.cached(*args, **kwargs)
def test_memory_kwarg():
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
for test in check_identity_lazy(g, accumulator):
yield test
memory = Memory(cachedir=env["dir"], verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
nose.tools.assert_equal(g(l=30, m=2), 30)
def test_memory_kwarg():
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
for test in check_identity_lazy(g, accumulator):
yield test
memory = Memory(cachedir=env['dir'], verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
nose.tools.assert_equal(g(l=30, m=2), 30)
def test_memory_ignore_decorated(tmpdir):
" Test the ignore feature of memory on a decorated function "
memory = Memory(location=tmpdir.strpath, verbose=0)
accumulator = list()
def decorate(f):
@functools.wraps(f)
def wrapped(*args, **kwargs):
return f(*args, **kwargs)
return wrapped
@memory.cache(ignore=['y'])
@decorate
def z(x, y=1):
accumulator.append(1)
assert z.ignore == ['y']
z(0, y=1)
assert len(accumulator) == 1
z(0, y=1)
assert len(accumulator) == 1
z(0, y=2)
assert len(accumulator) == 1
def test_memory_warning_collision_detection(tmpdir):
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(location=tmpdir.strpath, verbose=0)
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
with warns(JobLibCollisionWarning) as warninfo:
a1(1)
b1(1)
a1(0)
assert len(warninfo) == 2
assert "cannot detect" in str(warninfo[0].message).lower()
def test_memory_warning_lambda_collisions(tmpdir):
# Check that multiple use of lambda will raise collisions
memory = Memory(location=tmpdir.strpath, verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warns(JobLibCollisionWarning) as warninfo:
assert a(0) == 0
assert b(1) == 2
assert a(1) == 1
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
assert len(warninfo) == 4
def test_memory_warning_collision_detection(tmpdir):
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(location=tmpdir.strpath, verbose=0)
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
with warns(JobLibCollisionWarning) as warninfo:
a1(1)
b1(1)
a1(0)
assert len(warninfo) == 2
assert "cannot detect" in str(warninfo[0].message).lower()
def test_memory_warning_lambda_collisions(tmpdir):
# Check that multiple use of lambda will raise collisions
memory = Memory(location=tmpdir.strpath, verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warns(JobLibCollisionWarning) as warninfo:
assert a(0) == 0
assert b(1) == 2
assert a(1) == 1
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
assert len(warninfo) == 4
def test_cachedir_deprecation_warning(tmpdir):
# verify the right deprecation warnings are raised when using cachedir
# option instead of new location parameter.
with warns(None) as w:
memory = Memory(location=tmpdir.strpath, cachedir=tmpdir, verbose=0)
assert memory.store_backend.location.startswith(tmpdir.strpath)
assert len(w) == 1
assert "You set both location and cachedir options" in str(w[-1].message)
with warns(None) as w:
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
assert memory.store_backend.location.startswith(tmpdir.strpath)
assert len(w) == 1
assert "cachedir option is deprecated since version" in str(w[-1].message)
def test_memory_name_collision():
" Check that name collisions with functions will raise warnings"
memory = Memory(cachedir=env['dir'], verbose=0)
@memory.cache
def name_collision(x):
""" A first function called name_collision
"""
return x
a = name_collision
@memory.cache
def name_collision(x):
""" A second function called name_collision
"""
return x
b = name_collision
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# This is a temporary workaround until we get rid of
# inspect.getargspec, see
# https://github.com/joblib/joblib/issues/247
warnings.simplefilter("ignore", DeprecationWarning)
a(1)
b(1)
assert len(w) == 1
assert "collision" in str(w[-1].message)
def test_memory_name_collision(tmpdir):
" Check that name collisions with functions will raise warnings"
memory = Memory(location=tmpdir.strpath, verbose=0)
@memory.cache
def name_collision(x):
""" A first function called name_collision
"""
return x
a = name_collision
@memory.cache
def name_collision(x):
""" A second function called name_collision
"""
return x
b = name_collision
with warns(JobLibCollisionWarning) as warninfo:
a(1)
b(1)
assert len(warninfo) == 1
assert "collision" in str(warninfo[0].message)
def test_memory_numpy(tmpdir, mmap_mode):
" Test memory with a function with numpy arrays."
accumulator = list()
def n(l=None):
accumulator.append(1)
return l
memory = Memory(location=tmpdir.strpath, mmap_mode=mmap_mode, verbose=0)
cached_n = memory.cache(n)
rnd = np.random.RandomState(0)
for i in range(3):
a = rnd.random_sample((10, 10))
for _ in range(3):
assert np.all(cached_n(a) == a)
assert len(accumulator) == i + 1
def test_cached_function_race_condition_when_persisting_output(tmpdir, capfd):
# Test race condition where multiple processes are writing into
# the same output.pkl. See
# https://github.com/joblib/joblib/issues/490 for more details.
memory = Memory(location=tmpdir.strpath)
func_cached = memory.cache(fast_func_with_complex_output)
Parallel(n_jobs=2)(delayed(func_cached)() for i in range(3))
stdout, stderr = capfd.readouterr()
# Checking both stdout and stderr (ongoing PR #434 may change
# logging destination) to make sure there is no exception while
# loading the results
exception_msg = 'Exception while loading results'
assert exception_msg not in stdout
assert exception_msg not in stderr
def test_cached_function_race_condition_when_persisting_output(tmpdir, capfd):
# Test race condition where multiple processes are writing into
# the same output.pkl. See
# https://github.com/joblib/joblib/issues/490 for more details.
memory = Memory(location=tmpdir.strpath)
func_cached = memory.cache(fast_func_with_complex_output)
Parallel(n_jobs=2)(delayed(func_cached)() for i in range(3))
stdout, stderr = capfd.readouterr()
# Checking both stdout and stderr (ongoing PR #434 may change
# logging destination) to make sure there is no exception while
# loading the results
exception_msg = 'Exception while loading results'
assert exception_msg not in stdout
assert exception_msg not in stderr
def main():
# Now test clearing
for compress in (False, True):
for mmap_mode in ('r', None):
memory = Memory(location=tmpdir.strpath,
verbose=10,
mmap_mode=mmap_mode,
compress=compress)
# First clear the cache directory, to check that our code can
# handle that
# NOTE: this line would raise an exception, as the database file is
# still open; we ignore the error since we want to test what
# happens if the directory disappears
shutil.rmtree(tmpdir.strpath, ignore_errors=True)
g = memory.cache(f)
yield from g(1)
g.clear(warn=False)
current_accumulator = len(accumulator)
out = yield from g(1)
assert len(accumulator) == current_accumulator + 1
# Also, check that Memory.eval works similarly
evaled = yield from memory.eval(f, 1)
assert evaled == out
assert len(accumulator) == current_accumulator + 1
# Now do a smoke test with a function defined in __main__, as the name
# mangling rules are more complex
f.__module__ = '__main__'
memory = Memory(location=tmpdir.strpath, verbose=0)
yield from memory.cache(f)(1)
def test_memory_recomputes_after_an_error_while_loading_results(
tmpdir, monkeypatch):
memory = Memory(location=tmpdir.strpath)
def func(arg):
# This makes sure that the timestamp returned by two calls of
# func are different. This is needed on Windows where
# time.time resolution may not be accurate enough
time.sleep(0.01)
return arg, time.time()
cached_func = memory.cache(func)
input_arg = 'arg'
arg, timestamp = cached_func(input_arg)
# Make sure the function is correctly cached
assert arg == input_arg
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
corrupt_single_cache_item(memory)
# Make sure that corrupting the file causes recomputation and that
# a warning is issued.
recorded_warnings = monkeypatch_cached_func_warn(cached_func, monkeypatch)
recomputed_arg, recomputed_timestamp = cached_func(arg)
assert len(recorded_warnings) == 1
exception_msg = 'Exception while loading results'
assert exception_msg in recorded_warnings[0]
assert recomputed_arg == arg
assert recomputed_timestamp > timestamp
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
corrupt_single_cache_item(memory)
reference = cached_func.call_and_shelve(arg)
try:
reference.get()
raise AssertionError(
"It normally not possible to load a corrupted"
" MemorizedResult"
)
except KeyError as e:
message = "is corrupted"
assert message in str(e.args)
def test_memory_recomputes_after_an_error_while_loading_results(
tmpdir, monkeypatch):
memory = Memory(location=tmpdir.strpath)
def func(arg):
# This makes sure that the timestamp returned by two calls of
# func are different. This is needed on Windows where
# time.time resolution may not be accurate enough
time.sleep(0.01)
return arg, time.time()
cached_func = memory.cache(func)
input_arg = 'arg'
arg, timestamp = cached_func(input_arg)
# Make sure the function is correctly cached
assert arg == input_arg
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
corrupt_single_cache_item(memory)
# Make sure that corrupting the file causes recomputation and that
# a warning is issued.
recorded_warnings = monkeypatch_cached_func_warn(cached_func, monkeypatch)
recomputed_arg, recomputed_timestamp = cached_func(arg)
assert len(recorded_warnings) == 1
exception_msg = 'Exception while loading results'
assert exception_msg in recorded_warnings[0]
assert recomputed_arg == arg
assert recomputed_timestamp > timestamp
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
corrupt_single_cache_item(memory)
reference = cached_func.call_and_shelve(arg)
try:
reference.get()
raise AssertionError(
"It normally not possible to load a corrupted"
" MemorizedResult"
)
except KeyError as e:
message = "is corrupted"
assert message in str(e.args)
def test_memory_in_memory_function_code_change():
_function_to_cache.__code__ = _sum.__code__
mem = Memory(cachedir=env["dir"], verbose=0)
f = mem.cache(_function_to_cache)
nose.tools.assert_equal(f(1, 2), 3)
nose.tools.assert_equal(f(1, 2), 3)
with warnings.catch_warnings(record=True):
# ignore name collision warnings
warnings.simplefilter("always")
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
nose.tools.assert_equal(f(1, 2), 2)
nose.tools.assert_equal(f(1, 2), 2)
def test_memory_numpy(tmpdir, mmap_mode):
" Test memory with a function with numpy arrays."
accumulator = list()
def n(l=None):
accumulator.append(1)
return l
memory = Memory(location=tmpdir.strpath, mmap_mode=mmap_mode,
verbose=0)
cached_n = memory.cache(n)
rnd = np.random.RandomState(0)
for i in range(3):
a = rnd.random_sample((10, 10))
for _ in range(3):
assert np.all(cached_n(a) == a)
assert len(accumulator) == i + 1
def test_memory_in_memory_function_code_change():
_function_to_cache.__code__ = _sum.__code__
mem = Memory(cachedir=env['dir'], verbose=0)
f = mem.cache(_function_to_cache)
nose.tools.assert_equal(f(1, 2), 3)
nose.tools.assert_equal(f(1, 2), 3)
with warnings.catch_warnings(record=True):
# ignore name collision warnings
warnings.simplefilter("always")
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
nose.tools.assert_equal(f(1, 2), 2)
nose.tools.assert_equal(f(1, 2), 2)
def test_call_and_shelve(tmpdir):
# Test MemorizedFunc outputting a reference to cache.
for func, Result in zip((
MemorizedFunc(f, tmpdir.strpath),
NotMemorizedFunc(f),
Memory(location=tmpdir.strpath, verbose=0).cache(f),
Memory(location=None).cache(f),
), (MemorizedResult, NotMemorizedResult, MemorizedResult,
NotMemorizedResult)):
assert func(2) == 5
result = func.call_and_shelve(2)
assert isinstance(result, Result)
assert result.get() == 5
result.clear()
with raises(KeyError):
result.get()
result.clear() # Do nothing if there is no cache.
def test_call_and_shelve():
"""Test MemorizedFunc outputting a reference to cache.
"""
for func, Result in zip((
MemorizedFunc(f, env['dir']),
NotMemorizedFunc(f),
Memory(cachedir=env['dir']).cache(f),
Memory(cachedir=None).cache(f),
), (MemorizedResult, NotMemorizedResult, MemorizedResult,
NotMemorizedResult)):
nose.tools.assert_equal(func(2), 5)
result = func.call_and_shelve(2)
nose.tools.assert_true(isinstance(result, Result))
nose.tools.assert_equal(result.get(), 5)
result.clear()
nose.tools.assert_raises(KeyError, result.get)
result.clear() # Do nothing if there is no cache.
def test_persistence():
# Test the memorized functions can be pickled and restored.
memory = Memory(cachedir=env["dir"], verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
output_dir, _ = g.get_output_dir(1)
yield nose.tools.assert_equal, output, h.load_output(output_dir)
memory2 = pickle.loads(pickle.dumps(memory))
yield nose.tools.assert_equal, memory.cachedir, memory2.cachedir
# Smoke test that pickling a memory with cachedir=None works
memory = Memory(cachedir=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_memory_numpy():
" Test memory with a function with numpy arrays."
# Check with memmapping and without.
for mmap_mode in (None, 'r'):
accumulator = list()
def n(l=None):
accumulator.append(1)
return l
memory = Memory(cachedir=env['dir'], mmap_mode=mmap_mode, verbose=0)
memory.clear(warn=False)
cached_n = memory.cache(n)
rnd = np.random.RandomState(0)
for i in range(3):
a = rnd.random_sample((10, 10))
for _ in range(3):
yield nose.tools.assert_true, np.all(cached_n(a) == a)
yield nose.tools.assert_equal, len(accumulator), i + 1
def test_check_call_in_cache(tmpdir):
for func in (MemorizedFunc(f, tmpdir.strpath),
Memory(location=tmpdir.strpath, verbose=0).cache(f)):
result = func.check_call_in_cache(2)
assert not result
assert isinstance(result, bool)
assert func(2) == 5
result = func.check_call_in_cache(2)
assert result
assert isinstance(result, bool)
func.clear()
def test_persistence(tmpdir):
# Test the memorized functions can be pickled and restored.
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
output_dir, _ = h.get_output_dir(1)
func_name = _get_func_fullname(f)
assert output == _load_output(output_dir, func_name)
memory2 = pickle.loads(pickle.dumps(memory))
assert memory.cachedir == memory2.cachedir
# Smoke test that pickling a memory with cachedir=None works
memory = Memory(cachedir=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_memory_numpy_check_mmap_mode(tmpdir):
"""Check that mmap_mode is respected even at the first call"""
memory = Memory(cachedir=tmpdir.strpath, mmap_mode='r', verbose=0)
memory.clear(warn=False)
@memory.cache()
def twice(a):
return a * 2
a = np.ones(3)
b = twice(a)
c = twice(a)
assert isinstance(c, np.memmap)
assert c.mode == 'r'
assert isinstance(b, np.memmap)
assert b.mode == 'r'
def test_memory_numpy_check_mmap_mode():
"""Check that mmap_mode is respected even at the first call"""
memory = Memory(cachedir=env['dir'], mmap_mode='r', verbose=0)
memory.clear(warn=False)
@memory.cache()
def twice(a):
return a * 2
a = np.ones(3)
b = twice(a)
c = twice(a)
nose.tools.assert_true(isinstance(c, np.memmap))
nose.tools.assert_equal(c.mode, 'r')
nose.tools.assert_true(isinstance(b, np.memmap))
nose.tools.assert_equal(b.mode, 'r')
def test_partial_decoration(tmpdir, ignore, verbose, mmap_mode):
"Check cache may be called with kwargs before decorating"
memory = Memory(location=tmpdir.strpath, verbose=0)
@memory.cache(ignore=ignore, verbose=verbose, mmap_mode=mmap_mode)
def z(x):
pass
assert z.ignore == ignore
assert z._verbose == verbose
assert z.mmap_mode == mmap_mode
def test_memory_recomputes_after_an_error_why_loading_results(
tmpdir, monkeypatch):
memory = Memory(location=tmpdir.strpath)
def func(arg):
# This makes sure that the timestamp returned by two calls of
# func are different. This is needed on Windows where
# time.time resolution may not be accurate enough
time.sleep(0.01)
return arg, time.time()
cached_func = memory.cache(func)
input_arg = 'arg'
arg, timestamp = cached_func(input_arg)
# Make sure the function is correctly cached
assert arg == input_arg
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
single_cache_item, = memory.store_backend.get_items()
output_filename = os.path.join(single_cache_item.path, 'output.pkl')
with open(output_filename, 'w') as f:
f.write('garbage')
recorded_warnings = []
def append_to_record(item):
recorded_warnings.append(item)
# Make sure that corrupting the file causes recomputation and that
# a warning is issued. Need monkeypatch because pytest does not
# capture stdlib logging output (see
# https://github.com/pytest-dev/pytest/issues/2079)
monkeypatch.setattr(cached_func, 'warn', append_to_record)
recomputed_arg, recomputed_timestamp = cached_func(arg)
assert len(recorded_warnings) == 1
exception_msg = 'Exception while loading results'
assert exception_msg in recorded_warnings[0]
assert recomputed_arg == arg
assert recomputed_timestamp > timestamp
def test_cached_function_race_condition_when_persisting_output_2(
tmpdir, capfd):
# Test race condition in first attempt at solving
# https://github.com/joblib/joblib/issues/490. The race condition
# was due to the delay between seeing the cache directory created
# (interpreted as the result being cached) and the output.pkl being
# pickled.
memory = Memory(location=tmpdir.strpath)
func_cached = memory.cache(fast_func_with_conditional_complex_output)
Parallel(n_jobs=2)(delayed(func_cached)(True if i % 2 == 0 else False)
for i in range(3))
stdout, stderr = capfd.readouterr()
# Checking both stdout and stderr (ongoing PR #434 may change
# logging destination) to make sure there is no exception while
# loading the results
exception_msg = 'Exception while loading results'
assert exception_msg not in stdout
assert exception_msg not in stderr
def cached_func(tmpdir_factory):
# Create a Memory object to test decorated functions.
# We should be careful not to call the decorated functions, so that
# cache directories are not created in the temp dir.
cachedir = tmpdir_factory.mktemp("joblib_test_func_inspect")
mem = Memory(cachedir.strpath)
@mem.cache
def cached_func_inner(x):
return x
return cached_func_inner
def test_deprecated_cachedir_behaviour(tmpdir):
# verify the right deprecation warnings are raised when using cachedir
# option instead of new location parameter.
with warns(None) as w:
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
assert memory.store_backend.location.startswith(tmpdir.strpath)
assert len(w) == 1
assert "The 'cachedir' parameter has been deprecated" in str(w[-1].message)
with warns(None) as w:
memory = Memory()
assert memory.cachedir is None
assert len(w) == 1
assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message)
error_regex = """You set both "location='.+ and "cachedir='.+"""
with raises(ValueError, match=error_regex):
memory = Memory(location=tmpdir.strpath, cachedir=tmpdir.strpath,
verbose=0)
def test_memory_warning_collision_detection():
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(cachedir=env['dir'], verbose=0)
# For isolation with other tests
memory.clear()
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
if not hasattr(warnings, 'catch_warnings'):
# catch_warnings is new in Python 2.6
return
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# This is a temporary workaround until we get rid of
# inspect.getargspec, see
# https://github.com/joblib/joblib/issues/247
warnings.simplefilter("ignore", DeprecationWarning)
a1(1)
b1(1)
a1(0)
yield nose.tools.assert_equal, len(w), 2
yield nose.tools.assert_true, \
"cannot detect" in str(w[-1].message).lower()
def test_memory_arg_lambda():
" Test memory with a lambda argument."
memory = Memory(cachedir=env['dir'], verbose=0)
memory.clear(warn=False)
accum = {'value': 0}
@memory.cache()
def run_func(func):
accum['value'] += 1
return func()
lambda_1 = lambda: 1
lambda_2 = lambda: 2
a = run_func(lambda_1)
b = run_func(lambda_1)
c = run_func(lambda_2)
nose.tools.assert_equal(a, 1)
nose.tools.assert_equal(b, 1)
nose.tools.assert_equal(c, 2)
nose.tools.assert_equal(accum['value'], 2)
