def test_memory_integration():
""" Simple test of memory lazy evaluation.
"""
accumulator = list()
# Rmk: this function has the same name than a module-level function,
# thus it serves as a test to see that both are identified
# as different.
def f(l):
accumulator.append(1)
return l
for test in check_identity_lazy(f, accumulator):
yield test
# Now test clearing
for compress in (False, True):
for mmap_mode in ('r', None):
# We turn verbosity on to smoke test the verbosity code, however,
# we capture it, as it is ugly
try:
# To smoke-test verbosity, we capture stdout
orig_stdout = sys.stdout
orig_stderr = sys.stdout
if sys.version_info[0] == 3:
sys.stderr = io.StringIO()
sys.stderr = io.StringIO()
else:
sys.stdout = io.BytesIO()
sys.stderr = io.BytesIO()
memory = Memory(cachedir=env['dir'], verbose=10,
mmap_mode=mmap_mode, compress=compress)
# First clear the cache directory, to check that our code can
# handle that
# NOTE: this line would raise an exception, as the database file is
# still open; we ignore the error since we want to test what
# happens if the directory disappears
shutil.rmtree(env['dir'], ignore_errors=True)
g = memory.cache(f)
g(1)
g.clear(warn=False)
current_accumulator = len(accumulator)
out = g(1)
finally:
sys.stdout = orig_stdout
sys.stderr = orig_stderr
yield nose.tools.assert_equal, len(accumulator), \
current_accumulator + 1
# Also, check that Memory.eval works similarly
yield nose.tools.assert_equal, memory.eval(f, 1), out
yield nose.tools.assert_equal, len(accumulator), \
current_accumulator + 1
# Now do a smoke test with a function defined in __main__, as the name
# mangling rules are more complex
f.__module__ = '__main__'
memory = Memory(cachedir=env['dir'], verbose=0)
memory.cache(f)(1)
def test_memory_warning_collision_detection():
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(cachedir=env['dir'], verbose=0)
# For isolation with other tests
memory.clear()
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
if not hasattr(warnings, 'catch_warnings'):
# catch_warnings is new in Python 2.6
return
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# This is a temporary workaround until we get rid of
# inspect.getargspec, see
# https://github.com/joblib/joblib/issues/247
warnings.simplefilter("ignore", DeprecationWarning)
a1(1)
b1(1)
a1(0)
yield nose.tools.assert_equal, len(w), 2
yield nose.tools.assert_true, \
"cannot detect" in str(w[-1].message).lower()
def test_memory_warning_collision_detection():
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(cachedir=env['dir'], verbose=0)
# For isolation with other tests
memory.clear()
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
if not hasattr(warnings, 'catch_warnings'):
# catch_warnings is new in Python 2.6
return
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# This is a temporary workaround until we get rid of
# inspect.getargspec, see
# https://github.com/joblib/joblib/issues/247
warnings.simplefilter("ignore", DeprecationWarning)
a1(1)
b1(1)
a1(0)
yield nose.tools.assert_equal, len(w), 2
yield nose.tools.assert_true, \
"cannot detect" in str(w[-1].message).lower()
def test_memory_func_with_kwonly_args():
mem = Memory(cachedir=env['dir'], verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
nose.tools.assert_equal(func_cached(1, 2, kw1=3), (1, 2, 3, 'kw2'))
# Making sure that providing a keyword-only argument by
# position raises an exception
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached, 1, 2, 3, {'kw2': 4})
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached, 1, 2, 3, {'kw2': 4})
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2=4), (1, 2, 3, 4))
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2='ignored'),
(1, 2, 3, 4))
def test_memory_func_with_kwonly_args():
mem = Memory(cachedir=env['dir'], verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
nose.tools.assert_equal(func_cached(1, 2, kw1=3), (1, 2, 3, 'kw2'))
# Making sure that providing a keyword-only argument by
# position raises an exception
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached,
1, 2, 3, {'kw2': 4})
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
assert_raises_regex(
ValueError,
"Keyword-only parameter 'kw1' was passed as positional parameter",
func_cached,
1, 2, 3, {'kw2': 4})
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2=4), (1, 2, 3, 4))
nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2='ignored'), (1, 2, 3, 4))
def test_memory_func_with_kwonly_args(tmpdir):
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
func_cached = mem.cache(func_with_kwonly_args)
assert func_cached(1, 2, kw1=3) == (1, 2, 3, 'kw2')
# Making sure that providing a keyword-only argument by
# position raises an exception
with raises(ValueError) as excinfo:
func_cached(1, 2, 3, kw2=4)
excinfo.match("Keyword-only parameter 'kw1' was passed as positional "
"parameter")
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
with raises(ValueError) as excinfo:
func_cached(1, 2, 3, kw2=4)
excinfo.match("Keyword-only parameter 'kw1' was passed as positional "
"parameter")
# Test 'ignore' parameter
func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2'])
assert func_cached(1, 2, kw1=3, kw2=4) == (1, 2, 3, 4)
assert func_cached(1, 2, kw1=3, kw2='ignored') == (1, 2, 3, 4)
def main():
# Now test clearing
for compress in (False, True):
for mmap_mode in ('r', None):
memory = Memory(location=tmpdir.strpath,
verbose=10,
mmap_mode=mmap_mode,
compress=compress)
# First clear the cache directory, to check that our code can
# handle that
# NOTE: this line would raise an exception, as the database file is
# still open; we ignore the error since we want to test what
# happens if the directory disappears
shutil.rmtree(tmpdir.strpath, ignore_errors=True)
g = memory.cache(f)
yield from g(1)
g.clear(warn=False)
current_accumulator = len(accumulator)
out = yield from g(1)
assert len(accumulator) == current_accumulator + 1
# Also, check that Memory.eval works similarly
evaled = yield from memory.eval(f, 1)
assert evaled == out
assert len(accumulator) == current_accumulator + 1
# Now do a smoke test with a function defined in __main__, as the name
# mangling rules are more complex
f.__module__ = '__main__'
memory = Memory(location=tmpdir.strpath, verbose=0)
yield from memory.cache(f)(1)
def test_memory_func_with_kwonly_args(tmpdir):
memory = Memory(location=tmpdir.strpath, verbose=0)
func_cached = memory.cache(func_with_kwonly_args)
assert func_cached(1, 2, kw1=3) == (1, 2, 3, 'kw2')
# Making sure that providing a keyword-only argument by
# position raises an exception
with raises(ValueError) as excinfo:
func_cached(1, 2, 3, kw2=4)
excinfo.match("Keyword-only parameter 'kw1' was passed as positional "
"parameter")
# Keyword-only parameter passed by position with cached call
# should still raise ValueError
func_cached(1, 2, kw1=3, kw2=4)
with raises(ValueError) as excinfo:
func_cached(1, 2, 3, kw2=4)
excinfo.match("Keyword-only parameter 'kw1' was passed as positional "
"parameter")
# Test 'ignore' parameter
func_cached = memory.cache(func_with_kwonly_args, ignore=['kw2'])
assert func_cached(1, 2, kw1=3, kw2=4) == (1, 2, 3, 4)
assert func_cached(1, 2, kw1=3, kw2='ignored') == (1, 2, 3, 4)
def test_memory_file_modification(capsys):
# Test that modifying a Python file after loading it does not lead to
# Recomputation
dir_name = os.path.join(env['dir'], 'tmp_import')
if not os.path.exists(dir_name):
os.mkdir(dir_name)
filename = os.path.join(dir_name, 'tmp_joblib_.py')
content = 'def f(x):\n print(x)\n return x\n'
with open(filename, 'w') as module_file:
module_file.write(content)
# Load the module:
sys.path.append(dir_name)
import tmp_joblib_ as tmp
mem = Memory(cachedir=env['dir'], verbose=0)
f = mem.cache(tmp.f)
# First call f a few times
f(1)
f(2)
f(1)
# Now modify the module where f is stored without modifying f
with open(filename, 'w') as module_file:
module_file.write('\n\n' + content)
# And call f a couple more times
f(1)
f(1)
# Flush the .pyc files
shutil.rmtree(dir_name)
os.mkdir(dir_name)
# Now modify the module where f is stored, modifying f
content = 'def f(x):\n print("x=%s" % x)\n return x\n'
with open(filename, 'w') as module_file:
module_file.write(content)
# And call f more times prior to reloading: the cache should not be
# invalidated at this point as the active function definition has not
# changed in memory yet.
f(1)
f(1)
# Now reload
sys.stdout.write('Reloading\n')
sys.modules.pop('tmp_joblib_')
import tmp_joblib_ as tmp
f = mem.cache(tmp.f)
# And call f more times
f(1)
f(1)
out, err = capsys.readouterr()
assert out == '1\n2\nReloading\nx=1\n'
def test_memory_file_modification(capsys, tmpdir):
# Test that modifying a Python file after loading it does not lead to
# Recomputation
dir_name = tmpdir.mkdir('tmp_import').strpath
filename = os.path.join(dir_name, 'tmp_joblib_.py')
content = 'def f(x):\n print(x)\n return x\n'
with open(filename, 'w') as module_file:
module_file.write(content)
# Load the module:
sys.path.append(dir_name)
import tmp_joblib_ as tmp
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
f = mem.cache(tmp.f)
# First call f a few times
f(1)
f(2)
f(1)
# Now modify the module where f is stored without modifying f
with open(filename, 'w') as module_file:
module_file.write('\n\n' + content)
# And call f a couple more times
f(1)
f(1)
# Flush the .pyc files
shutil.rmtree(dir_name)
os.mkdir(dir_name)
# Now modify the module where f is stored, modifying f
content = 'def f(x):\n print("x=%s" % x)\n return x\n'
with open(filename, 'w') as module_file:
module_file.write(content)
# And call f more times prior to reloading: the cache should not be
# invalidated at this point as the active function definition has not
# changed in memory yet.
f(1)
f(1)
# Now reload
sys.stdout.write('Reloading\n')
sys.modules.pop('tmp_joblib_')
import tmp_joblib_ as tmp
f = mem.cache(tmp.f)
# And call f more times
f(1)
f(1)
out, err = capsys.readouterr()
assert out == '1\n2\nReloading\nx=1\n'
def test_call_and_shelve_lazily_load_stored_result(tmpdir):
"""Check call_and_shelve only load stored data if needed."""
test_access_time_file = tmpdir.join('test_access')
test_access_time_file.write('test_access')
test_access_time = os.stat(test_access_time_file.strpath).st_atime
# check file system access time stats resolution is lower than test wait
# timings.
time.sleep(0.5)
assert test_access_time_file.read() == 'test_access'
if test_access_time == os.stat(test_access_time_file.strpath).st_atime:
# Skip this test when access time cannot be retrieved with enough
# precision from the file system (e.g. NTFS on windows).
pytest.skip("filesystem does not support fine-grained access time "
"attribute")
memory = Memory(location=tmpdir.strpath, verbose=0)
func = memory.cache(f)
func_id, argument_hash = func._get_output_identifiers(2)
result_path = os.path.join(memory.store_backend.location,
func_id, argument_hash, 'output.pkl')
assert func(2) == 5
first_access_time = os.stat(result_path).st_atime
time.sleep(1)
# Should not access the stored data
result = func.call_and_shelve(2)
assert isinstance(result, MemorizedResult)
assert os.stat(result_path).st_atime == first_access_time
time.sleep(1)
# Read the stored data => last access time is greater than first_access
assert result.get() == 5
assert os.stat(result_path).st_atime > first_access_time
def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(location=tmpdir.strpath, verbose=0)
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
func_id = _build_func_identifier(f)
location = os.path.join(g.store_backend.location, func_id)
assert location == path
assert os.path.exists(path)
assert memory.location == os.path.dirname(g.store_backend.location)
with warns(DeprecationWarning) as w:
assert memory.cachedir == g.store_backend.location
assert len(w) == 1
assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
func_id, args_id = g._get_output_identifiers(1)
output_dir = os.path.join(g.store_backend.location, func_id, args_id)
a = g(1)
assert os.path.exists(output_dir)
os.remove(os.path.join(output_dir, 'output.pkl'))
assert a == g(1)
def test_memory_warning_collision_detection(tmpdir):
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(location=tmpdir.strpath, verbose=0)
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
with warns(JobLibCollisionWarning) as warninfo:
a1(1)
b1(1)
a1(0)
assert len(warninfo) == 2
assert "cannot detect" in str(warninfo[0].message).lower()
def test_memory_warning_lambda_collisions(tmpdir):
# Check that multiple use of lambda will raise collisions
memory = Memory(location=tmpdir.strpath, verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warns(JobLibCollisionWarning) as warninfo:
assert a(0) == 0
assert b(1) == 2
assert a(1) == 1
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
assert len(warninfo) == 4
def test_memory_pickle_dump_load(tmpdir, memory_kwargs):
memory = Memory(location=tmpdir.strpath, **memory_kwargs)
memory_reloaded = pickle.loads(pickle.dumps(memory))
# Compare Memory instance before and after pickle roundtrip
compare(memory.store_backend, memory_reloaded.store_backend)
compare(memory, memory_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(memory) == hash(memory_reloaded)
func_cached = memory.cache(f)
func_cached_reloaded = pickle.loads(pickle.dumps(func_cached))
# Compare MemorizedFunc instance before/after pickle roundtrip
compare(func_cached.store_backend, func_cached_reloaded.store_backend)
compare(func_cached, func_cached_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(func_cached) == hash(func_cached_reloaded)
# Compare MemorizedResult instance before/after pickle roundtrip
memorized_result = func_cached.call_and_shelve(1)
memorized_result_reloaded = pickle.loads(pickle.dumps(memorized_result))
compare(memorized_result.store_backend,
memorized_result_reloaded.store_backend)
compare(memorized_result, memorized_result_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(memorized_result) == hash(memorized_result_reloaded)
def test_memory_objects_repr(tmpdir):
# Verify printable reprs of MemorizedResult, MemorizedFunc and Memory.
def my_func(a, b):
return a + b
memory = Memory(location=tmpdir.strpath, verbose=0)
memorized_func = memory.cache(my_func)
memorized_func_repr = 'MemorizedFunc(func={func}, location={location})'
assert str(memorized_func) == memorized_func_repr.format(
func=my_func,
location=memory.store_backend.location)
memorized_result = memorized_func.call_and_shelve(42, 42)
memorized_result_repr = ('MemorizedResult(location="{location}", '
'func="{func}", args_id="{args_id}")')
assert str(memorized_result) == memorized_result_repr.format(
location=memory.store_backend.location,
func=memorized_result.func_id,
args_id=memorized_result.args_id)
assert str(memory) == 'Memory(location={location})'.format(
location=memory.store_backend.location)
def test_memory_recomputes_after_an_error_while_loading_results(
tmpdir, monkeypatch):
memory = Memory(location=tmpdir.strpath)
def func(arg):
# This makes sure that the timestamp returned by two calls of
# func are different. This is needed on Windows where
# time.time resolution may not be accurate enough
time.sleep(0.01)
return arg, time.time()
cached_func = memory.cache(func)
input_arg = 'arg'
arg, timestamp = cached_func(input_arg)
# Make sure the function is correctly cached
assert arg == input_arg
# Corrupting output.pkl to make sure that an error happens when
# loading the cached result
corrupt_single_cache_item(memory)
# Make sure that corrupting the file causes recomputation and that
# a warning is issued.
recorded_warnings = monkeypatch_cached_func_warn(cached_func, monkeypatch)
recomputed_arg, recomputed_timestamp = cached_func(arg)
assert len(recorded_warnings) == 1
exception_msg = 'Exception while loading results'
assert exception_msg in recorded_warnings[0]
assert recomputed_arg == arg
assert recomputed_timestamp > timestamp
def test_memory_pickle_dump_load(tmpdir, memory_kwargs):
memory = Memory(location=tmpdir.strpath, **memory_kwargs)
memory_reloaded = pickle.loads(pickle.dumps(memory))
# Compare Memory instance before and after pickle roundtrip
compare(memory.store_backend, memory_reloaded.store_backend)
compare(memory,
memory_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(memory) == hash(memory_reloaded)
func_cached = memory.cache(f)
func_cached_reloaded = pickle.loads(pickle.dumps(func_cached))
# Compare MemorizedFunc instance before/after pickle roundtrip
compare(func_cached.store_backend, func_cached_reloaded.store_backend)
compare(func_cached,
func_cached_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(func_cached) == hash(func_cached_reloaded)
# Compare MemorizedResult instance before/after pickle roundtrip
memorized_result = func_cached.call_and_shelve(1)
memorized_result_reloaded = pickle.loads(pickle.dumps(memorized_result))
compare(memorized_result.store_backend,
memorized_result_reloaded.store_backend)
compare(memorized_result,
memorized_result_reloaded,
ignored_attrs=set(['store_backend', 'timestamp']))
assert hash(memorized_result) == hash(memorized_result_reloaded)
def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
assert g._get_func_dir() == path
assert os.path.exists(path)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
dir, _ = g.get_output_dir(1)
a = g(1)
assert os.path.exists(dir)
os.remove(os.path.join(dir, 'output.pkl'))
assert a == g(1)
def test_memory_objects_repr(tmpdir):
# Verify printable reprs of MemorizedResult, MemorizedFunc and Memory.
def my_func(a, b):
return a + b
memory = Memory(location=tmpdir.strpath, verbose=0)
memorized_func = memory.cache(my_func)
memorized_func_repr = 'MemorizedFunc(func={func}, location={location})'
assert str(memorized_func) == memorized_func_repr.format(
func=my_func, location=memory.store_backend.location)
memorized_result = memorized_func.call_and_shelve(42, 42)
memorized_result_repr = ('MemorizedResult(location="{location}", '
'func="{func}", args_id="{args_id}")')
assert str(memorized_result) == memorized_result_repr.format(
location=memory.store_backend.location,
func=memorized_result.func_id,
args_id=memorized_result.args_id)
assert str(memory) == 'Memory(location={location})'.format(
location=memory.store_backend.location)
def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
memory.clear()
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
assert g._get_func_dir() == path
assert os.path.exists(path)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
dir, _ = g.get_output_dir(1)
a = g(1)
assert os.path.exists(dir)
os.remove(os.path.join(dir, 'output.pkl'))
assert a == g(1)
def test_memory_warning_collision_detection(tmpdir):
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(location=tmpdir.strpath, verbose=0)
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
with warns(JobLibCollisionWarning) as warninfo:
a1(1)
b1(1)
a1(0)
assert len(warninfo) == 2
assert "cannot detect" in str(warninfo[0].message).lower()
def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(location=tmpdir.strpath, verbose=0)
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
func_id = _build_func_identifier(f)
location = os.path.join(g.store_backend.location, func_id)
assert location == path
assert os.path.exists(path)
assert memory.location == os.path.dirname(g.store_backend.location)
with warns(DeprecationWarning) as w:
assert memory.cachedir == g.store_backend.location
assert len(w) == 1
assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
func_id, args_id = g._get_output_identifiers(1)
output_dir = os.path.join(g.store_backend.location, func_id, args_id)
a = g(1)
assert os.path.exists(output_dir)
os.remove(os.path.join(output_dir, 'output.pkl'))
assert a == g(1)
def test_call_and_shelve_lazily_load_stored_result(tmpdir):
"""Check call_and_shelve only load stored data if needed."""
test_access_time_file = tmpdir.join('test_access')
test_access_time_file.write('test_access')
test_access_time = os.stat(test_access_time_file.strpath).st_atime
# check file system access time stats resolution is lower than test wait
# timings.
time.sleep(0.5)
assert test_access_time_file.read() == 'test_access'
if test_access_time == os.stat(test_access_time_file.strpath).st_atime:
# Skip this test when access time cannot be retrieved with enough
# precision from the file system (e.g. NTFS on windows).
pytest.skip("filesystem does not support fine-grained access time "
"attribute")
memory = Memory(location=tmpdir.strpath, verbose=0)
func = memory.cache(f)
func_id, argument_hash = func._get_output_identifiers(2)
result_path = os.path.join(memory.store_backend.location, func_id,
argument_hash, 'output.pkl')
assert func(2) == 5
first_access_time = os.stat(result_path).st_atime
time.sleep(1)
# Should not access the stored data
result = func.call_and_shelve(2)
assert isinstance(result, MemorizedResult)
assert os.stat(result_path).st_atime == first_access_time
time.sleep(1)
# Read the stored data => last access time is greater than first_access
assert result.get() == 5
assert os.stat(result_path).st_atime > first_access_time
def test_func_dir():
# Test the creation of the memory cache directory for the function.
memory = Memory(cachedir=env['dir'], verbose=0)
memory.clear()
path = __name__.split('.')
path.append('f')
path = os.path.join(env['dir'], 'joblib', *path)
g = memory.cache(f)
# Test that the function directory is created on demand
yield nose.tools.assert_equal, g._get_func_dir(), path
yield nose.tools.assert_true, os.path.exists(path)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
yield nose.tools.assert_false, \
g._check_previous_func_code()
yield nose.tools.assert_true, \
os.path.exists(os.path.join(path, 'func_code.py'))
yield nose.tools.assert_true, \
g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
dir, _ = g.get_output_dir(1)
a = g(1)
yield nose.tools.assert_true, os.path.exists(dir)
os.remove(os.path.join(dir, 'output.pkl'))
yield nose.tools.assert_equal, a, g(1)
def main():
memory = Memory(location=None, verbose=0)
gg = memory.cache(ff)
for _ in range(4):
current_accumulator = len(accumulator)
yield from gg(1)
assert len(accumulator) == current_accumulator + 1
def test_func_dir():
# Test the creation of the memory cache directory for the function.
memory = Memory(cachedir=env["dir"], verbose=0)
memory.clear()
path = __name__.split(".")
path.append("f")
path = os.path.join(env["dir"], "joblib", *path)
g = memory.cache(f)
# Test that the function directory is created on demand
yield nose.tools.assert_equal, g._get_func_dir(), path
yield nose.tools.assert_true, os.path.exists(path)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
yield nose.tools.assert_false, g._check_previous_func_code()
yield nose.tools.assert_true, os.path.exists(os.path.join(path, "func_code.py"))
yield nose.tools.assert_true, g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
dir, _ = g.get_output_dir(1)
a = g(1)
yield nose.tools.assert_true, os.path.exists(dir)
os.remove(os.path.join(dir, "output.pkl"))
yield nose.tools.assert_equal, a, g(1)
def test_memory_warning_lambda_collisions(tmpdir):
# Check that multiple use of lambda will raise collisions
memory = Memory(location=tmpdir.strpath, verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warns(JobLibCollisionWarning) as warninfo:
assert a(0) == 0
assert b(1) == 2
assert a(1) == 1
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
assert len(warninfo) == 4
def test_memory_eval(tmpdir):
" Smoke test memory with a function with a function defined in an eval."
memory = Memory(location=tmpdir.strpath, verbose=0)
m = eval('lambda x: x')
mm = memory.cache(m)
assert mm(1) == 1
def test_memory_eval():
" Smoke test memory with a function with a function defined in an eval."
memory = Memory(cachedir=env['dir'], verbose=0)
m = eval('lambda x: x')
mm = memory.cache(m)
yield nose.tools.assert_equal, 1, mm(1)
def test_const_ndarray_as_cached_func_arg(tmpdir):
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
f_cache = memory.cache(f)
arr = const_ndarray(np.array([0, 1, 2]))
result = f_cache(arr) == f(arr)
assert np.any(result)
result = f_cache(arr) == f(arr)
assert np.any(result)
def test_memory_eval():
" Smoke test memory with a function with a function defined in an eval."
memory = Memory(cachedir=env["dir"], verbose=0)
m = eval("lambda x: x")
mm = memory.cache(m)
yield nose.tools.assert_equal, 1, mm(1)
def test_memory_eval(tmpdir):
" Smoke test memory with a function with a function defined in an eval."
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
m = eval('lambda x: x')
mm = memory.cache(m)
assert mm(1) == 1
def test_memory_integration():
""" Simple test of memory lazy evaluation.
"""
accumulator = list()
# Rmk: this function has the same name than a module-level function,
# thus it serves as a test to see that both are identified
# as different.
def f(l):
accumulator.append(1)
return l
for test in check_identity_lazy(f, accumulator):
yield test
# Now test clearing
for compress in (False, True):
for mmap_mode in ('r', None):
memory = Memory(cachedir=env['dir'],
verbose=10,
mmap_mode=mmap_mode,
compress=compress)
# First clear the cache directory, to check that our code can
# handle that
# NOTE: this line would raise an exception, as the database file is
# still open; we ignore the error since we want to test what
# happens if the directory disappears
shutil.rmtree(env['dir'], ignore_errors=True)
g = memory.cache(f)
g(1)
g.clear(warn=False)
current_accumulator = len(accumulator)
out = g(1)
yield assert_equal, len(accumulator), current_accumulator + 1
# Also, check that Memory.eval works similarly
yield assert_equal, memory.eval(f, 1), out
yield assert_equal, len(accumulator), current_accumulator + 1
# Now do a smoke test with a function defined in __main__, as the name
# mangling rules are more complex
f.__module__ = '__main__'
memory = Memory(cachedir=env['dir'], verbose=0)
memory.cache(f)(1)
def test_persistence():
# Test the memorized functions can be pickled and restored.
memory = Memory(cachedir=env['dir'], verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
output_dir, _ = g.get_output_dir(1)
yield nose.tools.assert_equal, output, h.load_output(output_dir)
memory2 = pickle.loads(pickle.dumps(memory))
yield nose.tools.assert_equal, memory.cachedir, memory2.cachedir
# Smoke test that pickling a memory with cachedir=None works
memory = Memory(cachedir=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_persistence():
# Test the memorized functions can be pickled and restored.
memory = Memory(cachedir=env["dir"], verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
output_dir, _ = g.get_output_dir(1)
yield nose.tools.assert_equal, output, h.load_output(output_dir)
memory2 = pickle.loads(pickle.dumps(memory))
yield nose.tools.assert_equal, memory.cachedir, memory2.cachedir
# Smoke test that pickling a memory with cachedir=None works
memory = Memory(cachedir=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_persistence(tmpdir):
# Test the memorized functions can be pickled and restored.
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
output_dir, _ = h.get_output_dir(1)
func_name = _get_func_fullname(f)
assert output == _load_output(output_dir, func_name)
memory2 = pickle.loads(pickle.dumps(memory))
assert memory.cachedir == memory2.cachedir
# Smoke test that pickling a memory with cachedir=None works
memory = Memory(cachedir=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_persistence(tmpdir):
# Test the memorized functions can be pickled and restored.
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
output_dir, _ = h.get_output_dir(1)
func_name = _get_func_fullname(f)
assert output == _load_output(output_dir, func_name)
memory2 = pickle.loads(pickle.dumps(memory))
assert memory.cachedir == memory2.cachedir
# Smoke test that pickling a memory with cachedir=None works
memory = Memory(cachedir=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_memory_warning_lambda_collisions():
# Check that multiple use of lambda will raise collisions
memory = Memory(cachedir=env["dir"], verbose=0)
# For isolation with other tests
memory.clear()
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
nose.tools.assert_equal(0, a(0))
nose.tools.assert_equal(2, b(1))
nose.tools.assert_equal(1, a(1))
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
nose.tools.assert_equal(len(w), 4)
def test_memory_integration(tmpdir):
""" Simple test of memory lazy evaluation.
"""
accumulator = list()
# Rmk: this function has the same name than a module-level function,
# thus it serves as a test to see that both are identified
# as different.
def f(l):
accumulator.append(1)
return l
check_identity_lazy(f, accumulator, tmpdir.strpath)
# Now test clearing
for compress in (False, True):
for mmap_mode in ('r', None):
memory = Memory(cachedir=tmpdir.strpath, verbose=10,
mmap_mode=mmap_mode, compress=compress)
# First clear the cache directory, to check that our code can
# handle that
# NOTE: this line would raise an exception, as the database file is
# still open; we ignore the error since we want to test what
# happens if the directory disappears
shutil.rmtree(tmpdir.strpath, ignore_errors=True)
g = memory.cache(f)
g(1)
g.clear(warn=False)
current_accumulator = len(accumulator)
out = g(1)
assert len(accumulator) == current_accumulator + 1
# Also, check that Memory.eval works similarly
assert memory.eval(f, 1) == out
assert len(accumulator) == current_accumulator + 1
# Now do a smoke test with a function defined in __main__, as the name
# mangling rules are more complex
f.__module__ = '__main__'
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
memory.cache(f)(1)
def test_persistence(tmpdir):
# Test the memorized functions can be pickled and restored.
memory = Memory(location=tmpdir.strpath, verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
func_id, args_id = h._get_output_identifiers(1)
output_dir = os.path.join(h.store_backend.location, func_id, args_id)
assert os.path.exists(output_dir)
assert output == h.store_backend.load_item([func_id, args_id])
memory2 = pickle.loads(pickle.dumps(memory))
assert memory.store_backend.location == memory2.store_backend.location
# Smoke test that pickling a memory with location=None works
memory = Memory(location=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def test_argument_change(tmpdir):
""" Check that if a function has a side effect in its arguments, it
should use the hash of changing arguments.
"""
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
func = mem.cache(count_and_append)
# call the function for the first time, is should cache it with
# argument x=[]
assert func() == 0
# the second time the argument is x=[None], which is not cached
# yet, so the functions should be called a second time
assert func() == 1
def test_argument_change(tmpdir):
""" Check that if a function has a side effect in its arguments, it
should use the hash of changing arguments.
"""
memory = Memory(location=tmpdir.strpath, verbose=0)
func = memory.cache(count_and_append)
# call the function for the first time, is should cache it with
# argument x=[]
assert func() == 0
# the second time the argument is x=[None], which is not cached
# yet, so the functions should be called a second time
assert func() == 1
def test_persistence(tmpdir):
# Test the memorized functions can be pickled and restored.
memory = Memory(location=tmpdir.strpath, verbose=0)
g = memory.cache(f)
output = g(1)
h = pickle.loads(pickle.dumps(g))
func_id, args_id = h._get_output_identifiers(1)
output_dir = os.path.join(h.store_backend.location, func_id, args_id)
assert os.path.exists(output_dir)
assert output == h.store_backend.load_item([func_id, args_id])
memory2 = pickle.loads(pickle.dumps(memory))
assert memory.store_backend.location == memory2.store_backend.location
# Smoke test that pickling a memory with location=None works
memory = Memory(location=None, verbose=0)
pickle.loads(pickle.dumps(memory))
g = memory.cache(f)
gp = pickle.loads(pickle.dumps(g))
gp(1)
def check_identity_lazy(func, accumulator, location):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(location=location, verbose=0)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
assert func(i) == i
assert len(accumulator) == i + 1
def check_identity_lazy(func, accumulator, location):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(location=location, verbose=0)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
assert func(i) == i
assert len(accumulator) == i + 1
def test_load(tmpdir):
" Test load function."
def g(a, b=1):
return a, b
memory = Memory(location=tmpdir.strpath, verbose=0)
g = memory.cache(g)
a, b = 1, 4
g(a, b=b)
assert (a, b) == g.load(a, b=b)
with raises(ValueError):
g.load(0, b=0)
def check_identity_lazy(func, accumulator):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(cachedir=env['dir'], verbose=0)
memory.clear(warn=False)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
yield nose.tools.assert_equal, func(i), i
yield nose.tools.assert_equal, len(accumulator), i + 1
def test_memory_in_memory_function_code_change(tmpdir):
_function_to_cache.__code__ = _sum.__code__
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
f = mem.cache(_function_to_cache)
assert f(1, 2) == 3
assert f(1, 2) == 3
with warns(JobLibCollisionWarning):
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
assert f(1, 2) == 2
assert f(1, 2) == 2
def test_memory_kwarg(tmpdir):
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
check_identity_lazy(g, accumulator, tmpdir.strpath)
memory = Memory(location=tmpdir.strpath, verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
assert g(l=30, m=2) == 30
def test_no_memory():
""" Test memory with location=None: no memoize """
accumulator = list()
def ff(l):
accumulator.append(1)
return l
memory = Memory(location=None, verbose=0)
gg = memory.cache(ff)
for _ in range(4):
current_accumulator = len(accumulator)
gg(1)
assert len(accumulator) == current_accumulator + 1
def check_identity_lazy(func, accumulator):
""" Given a function and an accumulator (a list that grows every
time the function is called), check that the function can be
decorated by memory to be a lazy identity.
"""
# Call each function with several arguments, and check that it is
# evaluated only once per argument.
memory = Memory(cachedir=env["dir"], verbose=0)
memory.clear(warn=False)
func = memory.cache(func)
for i in range(3):
for _ in range(2):
yield nose.tools.assert_equal, func(i), i
yield nose.tools.assert_equal, len(accumulator), i + 1
def test_no_memory():
""" Test memory with cachedir=None: no memoize """
accumulator = list()
def ff(l):
accumulator.append(1)
return l
mem = Memory(cachedir=None, verbose=0)
gg = mem.cache(ff)
for _ in range(4):
current_accumulator = len(accumulator)
gg(1)
assert len(accumulator) == current_accumulator + 1
def test_memory_kwarg(tmpdir):
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
check_identity_lazy(g, accumulator, tmpdir.strpath)
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
assert g(l=30, m=2) == 30
def check_identity_lazy_async(func, accumulator, location):
""" Similar to check_identity_lazy_async for coroutine functions"""
memory = Memory(location=location, verbose=0)
func = memory.cache(func)
@asyncio.coroutine
def main():
for i in range(3):
for _ in range(2):
value = yield from func(i)
assert value == i
assert len(accumulator) == i + 1
asyncio.get_event_loop().run_until_complete(main())
def test_memory_in_memory_function_code_change(tmpdir):
_function_to_cache.__code__ = _sum.__code__
memory = Memory(location=tmpdir.strpath, verbose=0)
f = memory.cache(_function_to_cache)
assert f(1, 2) == 3
assert f(1, 2) == 3
with warns(JobLibCollisionWarning):
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
assert f(1, 2) == 2
assert f(1, 2) == 2
def test_memory_kwarg():
" Test memory with a function with keyword arguments."
accumulator = list()
def g(l=None, m=1):
accumulator.append(1)
return l
for test in check_identity_lazy(g, accumulator):
yield test
memory = Memory(cachedir=env["dir"], verbose=0)
g = memory.cache(g)
# Smoke test with an explicit keyword argument:
nose.tools.assert_equal(g(l=30, m=2), 30)
def test_memory_warning_collision_detection():
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(cachedir=env["dir"], verbose=0)
# For isolation with other tests
memory.clear()
a1 = eval("lambda x: x")
a1 = memory.cache(a1)
b1 = eval("lambda x: x+1")
b1 = memory.cache(b1)
if not hasattr(warnings, "catch_warnings"):
# catch_warnings is new in Python 2.6
return
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
a1(1)
b1(1)
a1(0)
yield nose.tools.assert_equal, len(w), 2
yield nose.tools.assert_true, "cannot detect" in str(w[-1].message).lower()
def test_memory_warning_lambda_collisions():
# Check that multiple use of lambda will raise collisions
memory = Memory(cachedir=env['dir'], verbose=0)
# For isolation with other tests
memory.clear()
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# This is a temporary workaround until we get rid of
# inspect.getargspec, see
# https://github.com/joblib/joblib/issues/247
warnings.simplefilter("ignore", DeprecationWarning)
nose.tools.assert_equal(0, a(0))
nose.tools.assert_equal(2, b(1))
nose.tools.assert_equal(1, a(1))
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
nose.tools.assert_equal(len(w), 4)
def test_cached_function_race_condition_when_persisting_output(tmpdir, capfd):
# Test race condition where multiple processes are writing into
# the same output.pkl. See
# https://github.com/joblib/joblib/issues/490 for more details.
memory = Memory(location=tmpdir.strpath)
func_cached = memory.cache(fast_func_with_complex_output)
Parallel(n_jobs=2)(delayed(func_cached)() for i in range(3))
stdout, stderr = capfd.readouterr()
# Checking both stdout and stderr (ongoing PR #434 may change
# logging destination) to make sure there is no exception while
# loading the results
exception_msg = 'Exception while loading results'
assert exception_msg not in stdout
assert exception_msg not in stderr
