def test_func_dir(tmpdir): # Test the creation of the memory cache directory for the function. memory = Memory(cachedir=tmpdir.strpath, verbose=0) memory.clear() path = __name__.split('.') path.append('f') path = tmpdir.join('joblib', *path).strpath g = memory.cache(f) # Test that the function directory is created on demand assert g._get_func_dir() == path assert os.path.exists(path) # Test that the code is stored. # For the following test to be robust to previous execution, we clear # the in-memory store _FUNCTION_HASHES.clear() assert not g._check_previous_func_code() assert os.path.exists(os.path.join(path, 'func_code.py')) assert g._check_previous_func_code() # Test the robustness to failure of loading previous results. dir, _ = g.get_output_dir(1) a = g(1) assert os.path.exists(dir) os.remove(os.path.join(dir, 'output.pkl')) assert a == g(1)
def test_memory_func_with_kwonly_args(): mem = Memory(cachedir=env['dir'], verbose=0) func_cached = mem.cache(func_with_kwonly_args) nose.tools.assert_equal(func_cached(1, 2, kw1=3), (1, 2, 3, 'kw2')) # Making sure that providing a keyword-only argument by # position raises an exception assert_raises_regex( ValueError, "Keyword-only parameter 'kw1' was passed as positional parameter", func_cached, 1, 2, 3, {'kw2': 4}) # Keyword-only parameter passed by position with cached call # should still raise ValueError func_cached(1, 2, kw1=3, kw2=4) assert_raises_regex( ValueError, "Keyword-only parameter 'kw1' was passed as positional parameter", func_cached, 1, 2, 3, {'kw2': 4}) # Test 'ignore' parameter func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2']) nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2=4), (1, 2, 3, 4)) nose.tools.assert_equal(func_cached(1, 2, kw1=3, kw2='ignored'), (1, 2, 3, 4))
def test_func_dir(tmpdir): # Test the creation of the memory cache directory for the function. memory = Memory(location=tmpdir.strpath, verbose=0) path = __name__.split('.') path.append('f') path = tmpdir.join('joblib', *path).strpath g = memory.cache(f) # Test that the function directory is created on demand func_id = _build_func_identifier(f) location = os.path.join(g.store_backend.location, func_id) assert location == path assert os.path.exists(path) assert memory.location == os.path.dirname(g.store_backend.location) with warns(DeprecationWarning) as w: assert memory.cachedir == g.store_backend.location assert len(w) == 1 assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message) # Test that the code is stored. # For the following test to be robust to previous execution, we clear # the in-memory store _FUNCTION_HASHES.clear() assert not g._check_previous_func_code() assert os.path.exists(os.path.join(path, 'func_code.py')) assert g._check_previous_func_code() # Test the robustness to failure of loading previous results. func_id, args_id = g._get_output_identifiers(1) output_dir = os.path.join(g.store_backend.location, func_id, args_id) a = g(1) assert os.path.exists(output_dir) os.remove(os.path.join(output_dir, 'output.pkl')) assert a == g(1)
def test_func_dir(): # Test the creation of the memory cache directory for the function. memory = Memory(cachedir=env["dir"], verbose=0) memory.clear() path = __name__.split(".") path.append("f") path = os.path.join(env["dir"], "joblib", *path) g = memory.cache(f) # Test that the function directory is created on demand yield nose.tools.assert_equal, g._get_func_dir(), path yield nose.tools.assert_true, os.path.exists(path) # Test that the code is stored. # For the following test to be robust to previous execution, we clear # the in-memory store _FUNCTION_HASHES.clear() yield nose.tools.assert_false, g._check_previous_func_code() yield nose.tools.assert_true, os.path.exists(os.path.join(path, "func_code.py")) yield nose.tools.assert_true, g._check_previous_func_code() # Test the robustness to failure of loading previous results. dir, _ = g.get_output_dir(1) a = g(1) yield nose.tools.assert_true, os.path.exists(dir) os.remove(os.path.join(dir, "output.pkl")) yield nose.tools.assert_equal, a, g(1)
def test_call_and_shelve_lazily_load_stored_result(tmpdir): """Check call_and_shelve only load stored data if needed.""" test_access_time_file = tmpdir.join('test_access') test_access_time_file.write('test_access') test_access_time = os.stat(test_access_time_file.strpath).st_atime # check file system access time stats resolution is lower than test wait # timings. time.sleep(0.5) assert test_access_time_file.read() == 'test_access' if test_access_time == os.stat(test_access_time_file.strpath).st_atime: # Skip this test when access time cannot be retrieved with enough # precision from the file system (e.g. NTFS on windows). pytest.skip("filesystem does not support fine-grained access time " "attribute") memory = Memory(location=tmpdir.strpath, verbose=0) func = memory.cache(f) func_id, argument_hash = func._get_output_identifiers(2) result_path = os.path.join(memory.store_backend.location, func_id, argument_hash, 'output.pkl') assert func(2) == 5 first_access_time = os.stat(result_path).st_atime time.sleep(1) # Should not access the stored data result = func.call_and_shelve(2) assert isinstance(result, MemorizedResult) assert os.stat(result_path).st_atime == first_access_time time.sleep(1) # Read the stored data => last access time is greater than first_access assert result.get() == 5 assert os.stat(result_path).st_atime > first_access_time
def test_memory_warning_collision_detection(): # Check that collisions impossible to detect will raise appropriate # warnings. memory = Memory(cachedir=env['dir'], verbose=0) # For isolation with other tests memory.clear() a1 = eval('lambda x: x') a1 = memory.cache(a1) b1 = eval('lambda x: x+1') b1 = memory.cache(b1) if not hasattr(warnings, 'catch_warnings'): # catch_warnings is new in Python 2.6 return with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # This is a temporary workaround until we get rid of # inspect.getargspec, see # https://github.com/joblib/joblib/issues/247 warnings.simplefilter("ignore", DeprecationWarning) a1(1) b1(1) a1(0) yield nose.tools.assert_equal, len(w), 2 yield nose.tools.assert_true, \ "cannot detect" in str(w[-1].message).lower()
def test_memory_pickle_dump_load(tmpdir, memory_kwargs): memory = Memory(location=tmpdir.strpath, **memory_kwargs) memory_reloaded = pickle.loads(pickle.dumps(memory)) # Compare Memory instance before and after pickle roundtrip compare(memory.store_backend, memory_reloaded.store_backend) compare(memory, memory_reloaded, ignored_attrs=set(['store_backend', 'timestamp'])) assert hash(memory) == hash(memory_reloaded) func_cached = memory.cache(f) func_cached_reloaded = pickle.loads(pickle.dumps(func_cached)) # Compare MemorizedFunc instance before/after pickle roundtrip compare(func_cached.store_backend, func_cached_reloaded.store_backend) compare(func_cached, func_cached_reloaded, ignored_attrs=set(['store_backend', 'timestamp'])) assert hash(func_cached) == hash(func_cached_reloaded) # Compare MemorizedResult instance before/after pickle roundtrip memorized_result = func_cached.call_and_shelve(1) memorized_result_reloaded = pickle.loads(pickle.dumps(memorized_result)) compare(memorized_result.store_backend, memorized_result_reloaded.store_backend) compare(memorized_result, memorized_result_reloaded, ignored_attrs=set(['store_backend', 'timestamp'])) assert hash(memorized_result) == hash(memorized_result_reloaded)
def test_memory_objects_repr(tmpdir): # Verify printable reprs of MemorizedResult, MemorizedFunc and Memory. def my_func(a, b): return a + b memory = Memory(location=tmpdir.strpath, verbose=0) memorized_func = memory.cache(my_func) memorized_func_repr = 'MemorizedFunc(func={func}, location={location})' assert str(memorized_func) == memorized_func_repr.format( func=my_func, location=memory.store_backend.location) memorized_result = memorized_func.call_and_shelve(42, 42) memorized_result_repr = ('MemorizedResult(location="{location}", ' 'func="{func}", args_id="{args_id}")') assert str(memorized_result) == memorized_result_repr.format( location=memory.store_backend.location, func=memorized_result.func_id, args_id=memorized_result.args_id) assert str(memory) == 'Memory(location={location})'.format( location=memory.store_backend.location)
def test_memory_func_with_kwonly_args(tmpdir): mem = Memory(cachedir=tmpdir.strpath, verbose=0) func_cached = mem.cache(func_with_kwonly_args) assert func_cached(1, 2, kw1=3) == (1, 2, 3, 'kw2') # Making sure that providing a keyword-only argument by # position raises an exception with raises(ValueError) as excinfo: func_cached(1, 2, 3, kw2=4) excinfo.match("Keyword-only parameter 'kw1' was passed as positional " "parameter") # Keyword-only parameter passed by position with cached call # should still raise ValueError func_cached(1, 2, kw1=3, kw2=4) with raises(ValueError) as excinfo: func_cached(1, 2, 3, kw2=4) excinfo.match("Keyword-only parameter 'kw1' was passed as positional " "parameter") # Test 'ignore' parameter func_cached = mem.cache(func_with_kwonly_args, ignore=['kw2']) assert func_cached(1, 2, kw1=3, kw2=4) == (1, 2, 3, 4) assert func_cached(1, 2, kw1=3, kw2='ignored') == (1, 2, 3, 4)
def test_memory_arg_nested(): " Test memory with a nested function argument." memory = Memory(cachedir=env['dir'], verbose=0) memory.clear(warn=False) accum = {'value': 0} def func_1(): return 1 def func_2(): return 2 def decorator(func_): def decorated(): return func_() return decorated @memory.cache() def run_func(func): accum['value'] += 1 return func() a = run_func(decorator(func_1)) b = run_func(decorator(func_1)) c = run_func(decorator(func_2)) nose.tools.assert_equal(a, 1) nose.tools.assert_equal(b, 1) nose.tools.assert_equal(c, 2) nose.tools.assert_equal(accum['value'], 2)
def test_memory_eval(tmpdir): " Smoke test memory with a function with a function defined in an eval." memory = Memory(cachedir=tmpdir.strpath, verbose=0) m = eval('lambda x: x') mm = memory.cache(m) assert mm(1) == 1
def test_memory_eval(): " Smoke test memory with a function with a function defined in an eval." memory = Memory(cachedir=env["dir"], verbose=0) m = eval("lambda x: x") mm = memory.cache(m) yield nose.tools.assert_equal, 1, mm(1)
def test_call_and_shelve_argument_hash(tmpdir): # Verify that a warning is raised when accessing arguments_hash # attribute from MemorizedResult func = Memory(location=tmpdir.strpath, verbose=0).cache(f) result = func.call_and_shelve(2) assert isinstance(result, MemorizedResult) with warns(DeprecationWarning) as w: assert result.argument_hash == result.args_id assert len(w) == 1 assert "The 'argument_hash' attribute has been deprecated" \ in str(w[-1].message)
def test_memory_file_modification(capsys, tmpdir): # Test that modifying a Python file after loading it does not lead to # Recomputation dir_name = tmpdir.mkdir('tmp_import').strpath filename = os.path.join(dir_name, 'tmp_joblib_.py') content = 'def f(x):\n print(x)\n return x\n' with open(filename, 'w') as module_file: module_file.write(content) # Load the module: sys.path.append(dir_name) import tmp_joblib_ as tmp mem = Memory(cachedir=tmpdir.strpath, verbose=0) f = mem.cache(tmp.f) # First call f a few times f(1) f(2) f(1) # Now modify the module where f is stored without modifying f with open(filename, 'w') as module_file: module_file.write('\n\n' + content) # And call f a couple more times f(1) f(1) # Flush the .pyc files shutil.rmtree(dir_name) os.mkdir(dir_name) # Now modify the module where f is stored, modifying f content = 'def f(x):\n print("x=%s" % x)\n return x\n' with open(filename, 'w') as module_file: module_file.write(content) # And call f more times prior to reloading: the cache should not be # invalidated at this point as the active function definition has not # changed in memory yet. f(1) f(1) # Now reload sys.stdout.write('Reloading\n') sys.modules.pop('tmp_joblib_') import tmp_joblib_ as tmp f = mem.cache(tmp.f) # And call f more times f(1) f(1) out, err = capsys.readouterr() assert out == '1\n2\nReloading\nx=1\n'
def test_argument_change(tmpdir): """ Check that if a function has a side effect in its arguments, it should use the hash of changing arguments. """ mem = Memory(cachedir=tmpdir.strpath, verbose=0) func = mem.cache(count_and_append) # call the function for the first time, is should cache it with # argument x=[] assert func() == 0 # the second time the argument is x=[None], which is not cached # yet, so the functions should be called a second time assert func() == 1
def check_identity_lazy(func, accumulator, location): """ Given a function and an accumulator (a list that grows every time the function is called), check that the function can be decorated by memory to be a lazy identity. """ # Call each function with several arguments, and check that it is # evaluated only once per argument. memory = Memory(location=location, verbose=0) func = memory.cache(func) for i in range(3): for _ in range(2): assert func(i) == i assert len(accumulator) == i + 1
def test_memory_kwarg(tmpdir): " Test memory with a function with keyword arguments." accumulator = list() def g(l=None, m=1): accumulator.append(1) return l check_identity_lazy(g, accumulator, tmpdir.strpath) memory = Memory(location=tmpdir.strpath, verbose=0) g = memory.cache(g) # Smoke test with an explicit keyword argument: assert g(l=30, m=2) == 30
def test_no_memory(): """ Test memory with cachedir=None: no memoize """ accumulator = list() def ff(l): accumulator.append(1) return l mem = Memory(cachedir=None, verbose=0) gg = mem.cache(ff) for _ in range(4): current_accumulator = len(accumulator) gg(1) assert len(accumulator) == current_accumulator + 1
def test_no_memory(): """ Test memory with location=None: no memoize """ accumulator = list() def ff(l): accumulator.append(1) return l memory = Memory(location=None, verbose=0) gg = memory.cache(ff) for _ in range(4): current_accumulator = len(accumulator) gg(1) assert len(accumulator) == current_accumulator + 1
def check_identity_lazy(func, accumulator): """ Given a function and an accumulator (a list that grows every time the function is called), check that the function can be decorated by memory to be a lazy identity. """ # Call each function with several arguments, and check that it is # evaluated only once per argument. memory = Memory(cachedir=env['dir'], verbose=0) memory.clear(warn=False) func = memory.cache(func) for i in range(3): for _ in range(2): yield nose.tools.assert_equal, func(i), i yield nose.tools.assert_equal, len(accumulator), i + 1
def check_identity_lazy(func, accumulator): """ Given a function and an accumulator (a list that grows every time the function is called), check that the function can be decorated by memory to be a lazy identity. """ # Call each function with several arguments, and check that it is # evaluated only once per argument. memory = Memory(cachedir=env["dir"], verbose=0) memory.clear(warn=False) func = memory.cache(func) for i in range(3): for _ in range(2): yield nose.tools.assert_equal, func(i), i yield nose.tools.assert_equal, len(accumulator), i + 1
def test_memory_kwarg(tmpdir): " Test memory with a function with keyword arguments." accumulator = list() def g(l=None, m=1): accumulator.append(1) return l check_identity_lazy(g, accumulator, tmpdir.strpath) memory = Memory(cachedir=tmpdir.strpath, verbose=0) g = memory.cache(g) # Smoke test with an explicit keyword argument: assert g(l=30, m=2) == 30
def test_memory_in_memory_function_code_change(tmpdir): _function_to_cache.__code__ = _sum.__code__ memory = Memory(location=tmpdir.strpath, verbose=0) f = memory.cache(_function_to_cache) assert f(1, 2) == 3 assert f(1, 2) == 3 with warns(JobLibCollisionWarning): # Check that inline function modification triggers a cache invalidation _function_to_cache.__code__ = _product.__code__ assert f(1, 2) == 2 assert f(1, 2) == 2
def test_memory_in_memory_function_code_change(tmpdir): _function_to_cache.__code__ = _sum.__code__ mem = Memory(cachedir=tmpdir.strpath, verbose=0) f = mem.cache(_function_to_cache) assert f(1, 2) == 3 assert f(1, 2) == 3 with warns(JobLibCollisionWarning): # Check that inline function modification triggers a cache invalidation _function_to_cache.__code__ = _product.__code__ assert f(1, 2) == 2 assert f(1, 2) == 2
def test_no_memory(): """ Test memory with cachedir=None: no memoize """ accumulator = list() def ff(l): accumulator.append(1) return l mem = Memory(cachedir=None, verbose=0) gg = mem.cache(ff) for _ in range(4): current_accumulator = len(accumulator) gg(1) yield nose.tools.assert_equal, len(accumulator), \ current_accumulator + 1
def test_cached(tmpdir): " Test cached function." def g(a, b, c, d, e=None, f=1): return (a, b, c, d, e, f) memory = Memory(location=tmpdir.strpath, verbose=0) g = memory.cache(g) args = (1, 2, 3, 4) kwargs = {'e': 5, 'f': 6} g(*args, **kwargs) assert g.cached(*args, **kwargs) assert not g.cached(*args[::-1], **kwargs) g.clear() assert not g.cached(*args, **kwargs)
def test_memory_kwarg(): " Test memory with a function with keyword arguments." accumulator = list() def g(l=None, m=1): accumulator.append(1) return l for test in check_identity_lazy(g, accumulator): yield test memory = Memory(cachedir=env["dir"], verbose=0) g = memory.cache(g) # Smoke test with an explicit keyword argument: nose.tools.assert_equal(g(l=30, m=2), 30)
def test_memory_kwarg(): " Test memory with a function with keyword arguments." accumulator = list() def g(l=None, m=1): accumulator.append(1) return l for test in check_identity_lazy(g, accumulator): yield test memory = Memory(cachedir=env['dir'], verbose=0) g = memory.cache(g) # Smoke test with an explicit keyword argument: nose.tools.assert_equal(g(l=30, m=2), 30)
def test_memory_ignore_decorated(tmpdir): " Test the ignore feature of memory on a decorated function " memory = Memory(location=tmpdir.strpath, verbose=0) accumulator = list() def decorate(f): @functools.wraps(f) def wrapped(*args, **kwargs): return f(*args, **kwargs) return wrapped @memory.cache(ignore=['y']) @decorate def z(x, y=1): accumulator.append(1) assert z.ignore == ['y'] z(0, y=1) assert len(accumulator) == 1 z(0, y=1) assert len(accumulator) == 1 z(0, y=2) assert len(accumulator) == 1
def test_memory_warning_collision_detection(tmpdir): # Check that collisions impossible to detect will raise appropriate # warnings. memory = Memory(location=tmpdir.strpath, verbose=0) a1 = eval('lambda x: x') a1 = memory.cache(a1) b1 = eval('lambda x: x+1') b1 = memory.cache(b1) with warns(JobLibCollisionWarning) as warninfo: a1(1) b1(1) a1(0) assert len(warninfo) == 2 assert "cannot detect" in str(warninfo[0].message).lower()
def test_memory_warning_lambda_collisions(tmpdir): # Check that multiple use of lambda will raise collisions memory = Memory(location=tmpdir.strpath, verbose=0) a = lambda x: x a = memory.cache(a) b = lambda x: x + 1 b = memory.cache(b) with warns(JobLibCollisionWarning) as warninfo: assert a(0) == 0 assert b(1) == 2 assert a(1) == 1 # In recent Python versions, we can retrieve the code of lambdas, # thus nothing is raised assert len(warninfo) == 4
def test_cachedir_deprecation_warning(tmpdir): # verify the right deprecation warnings are raised when using cachedir # option instead of new location parameter. with warns(None) as w: memory = Memory(location=tmpdir.strpath, cachedir=tmpdir, verbose=0) assert memory.store_backend.location.startswith(tmpdir.strpath) assert len(w) == 1 assert "You set both location and cachedir options" in str(w[-1].message) with warns(None) as w: memory = Memory(cachedir=tmpdir.strpath, verbose=0) assert memory.store_backend.location.startswith(tmpdir.strpath) assert len(w) == 1 assert "cachedir option is deprecated since version" in str(w[-1].message)
def test_memory_name_collision(): " Check that name collisions with functions will raise warnings" memory = Memory(cachedir=env['dir'], verbose=0) @memory.cache def name_collision(x): """ A first function called name_collision """ return x a = name_collision @memory.cache def name_collision(x): """ A second function called name_collision """ return x b = name_collision with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") # This is a temporary workaround until we get rid of # inspect.getargspec, see # https://github.com/joblib/joblib/issues/247 warnings.simplefilter("ignore", DeprecationWarning) a(1) b(1) assert len(w) == 1 assert "collision" in str(w[-1].message)
def test_memory_name_collision(tmpdir): " Check that name collisions with functions will raise warnings" memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache def name_collision(x): """ A first function called name_collision """ return x a = name_collision @memory.cache def name_collision(x): """ A second function called name_collision """ return x b = name_collision with warns(JobLibCollisionWarning) as warninfo: a(1) b(1) assert len(warninfo) == 1 assert "collision" in str(warninfo[0].message)
def test_memory_numpy(tmpdir, mmap_mode): " Test memory with a function with numpy arrays." accumulator = list() def n(l=None): accumulator.append(1) return l memory = Memory(location=tmpdir.strpath, mmap_mode=mmap_mode, verbose=0) cached_n = memory.cache(n) rnd = np.random.RandomState(0) for i in range(3): a = rnd.random_sample((10, 10)) for _ in range(3): assert np.all(cached_n(a) == a) assert len(accumulator) == i + 1
def test_cached_function_race_condition_when_persisting_output(tmpdir, capfd): # Test race condition where multiple processes are writing into # the same output.pkl. See # https://github.com/joblib/joblib/issues/490 for more details. memory = Memory(location=tmpdir.strpath) func_cached = memory.cache(fast_func_with_complex_output) Parallel(n_jobs=2)(delayed(func_cached)() for i in range(3)) stdout, stderr = capfd.readouterr() # Checking both stdout and stderr (ongoing PR #434 may change # logging destination) to make sure there is no exception while # loading the results exception_msg = 'Exception while loading results' assert exception_msg not in stdout assert exception_msg not in stderr
def main(): # Now test clearing for compress in (False, True): for mmap_mode in ('r', None): memory = Memory(location=tmpdir.strpath, verbose=10, mmap_mode=mmap_mode, compress=compress) # First clear the cache directory, to check that our code can # handle that # NOTE: this line would raise an exception, as the database file is # still open; we ignore the error since we want to test what # happens if the directory disappears shutil.rmtree(tmpdir.strpath, ignore_errors=True) g = memory.cache(f) yield from g(1) g.clear(warn=False) current_accumulator = len(accumulator) out = yield from g(1) assert len(accumulator) == current_accumulator + 1 # Also, check that Memory.eval works similarly evaled = yield from memory.eval(f, 1) assert evaled == out assert len(accumulator) == current_accumulator + 1 # Now do a smoke test with a function defined in __main__, as the name # mangling rules are more complex f.__module__ = '__main__' memory = Memory(location=tmpdir.strpath, verbose=0) yield from memory.cache(f)(1)
def test_memory_recomputes_after_an_error_while_loading_results( tmpdir, monkeypatch): memory = Memory(location=tmpdir.strpath) def func(arg): # This makes sure that the timestamp returned by two calls of # func are different. This is needed on Windows where # time.time resolution may not be accurate enough time.sleep(0.01) return arg, time.time() cached_func = memory.cache(func) input_arg = 'arg' arg, timestamp = cached_func(input_arg) # Make sure the function is correctly cached assert arg == input_arg # Corrupting output.pkl to make sure that an error happens when # loading the cached result corrupt_single_cache_item(memory) # Make sure that corrupting the file causes recomputation and that # a warning is issued. recorded_warnings = monkeypatch_cached_func_warn(cached_func, monkeypatch) recomputed_arg, recomputed_timestamp = cached_func(arg) assert len(recorded_warnings) == 1 exception_msg = 'Exception while loading results' assert exception_msg in recorded_warnings[0] assert recomputed_arg == arg assert recomputed_timestamp > timestamp # Corrupting output.pkl to make sure that an error happens when # loading the cached result corrupt_single_cache_item(memory) reference = cached_func.call_and_shelve(arg) try: reference.get() raise AssertionError( "It normally not possible to load a corrupted" " MemorizedResult" ) except KeyError as e: message = "is corrupted" assert message in str(e.args)
def test_memory_in_memory_function_code_change(): _function_to_cache.__code__ = _sum.__code__ mem = Memory(cachedir=env["dir"], verbose=0) f = mem.cache(_function_to_cache) nose.tools.assert_equal(f(1, 2), 3) nose.tools.assert_equal(f(1, 2), 3) with warnings.catch_warnings(record=True): # ignore name collision warnings warnings.simplefilter("always") # Check that inline function modification triggers a cache invalidation _function_to_cache.__code__ = _product.__code__ nose.tools.assert_equal(f(1, 2), 2) nose.tools.assert_equal(f(1, 2), 2)
def test_memory_in_memory_function_code_change(): _function_to_cache.__code__ = _sum.__code__ mem = Memory(cachedir=env['dir'], verbose=0) f = mem.cache(_function_to_cache) nose.tools.assert_equal(f(1, 2), 3) nose.tools.assert_equal(f(1, 2), 3) with warnings.catch_warnings(record=True): # ignore name collision warnings warnings.simplefilter("always") # Check that inline function modification triggers a cache invalidation _function_to_cache.__code__ = _product.__code__ nose.tools.assert_equal(f(1, 2), 2) nose.tools.assert_equal(f(1, 2), 2)
def test_call_and_shelve(tmpdir): # Test MemorizedFunc outputting a reference to cache. for func, Result in zip(( MemorizedFunc(f, tmpdir.strpath), NotMemorizedFunc(f), Memory(location=tmpdir.strpath, verbose=0).cache(f), Memory(location=None).cache(f), ), (MemorizedResult, NotMemorizedResult, MemorizedResult, NotMemorizedResult)): assert func(2) == 5 result = func.call_and_shelve(2) assert isinstance(result, Result) assert result.get() == 5 result.clear() with raises(KeyError): result.get() result.clear() # Do nothing if there is no cache.
def test_call_and_shelve(): """Test MemorizedFunc outputting a reference to cache. """ for func, Result in zip(( MemorizedFunc(f, env['dir']), NotMemorizedFunc(f), Memory(cachedir=env['dir']).cache(f), Memory(cachedir=None).cache(f), ), (MemorizedResult, NotMemorizedResult, MemorizedResult, NotMemorizedResult)): nose.tools.assert_equal(func(2), 5) result = func.call_and_shelve(2) nose.tools.assert_true(isinstance(result, Result)) nose.tools.assert_equal(result.get(), 5) result.clear() nose.tools.assert_raises(KeyError, result.get) result.clear() # Do nothing if there is no cache.
def test_persistence(): # Test the memorized functions can be pickled and restored. memory = Memory(cachedir=env["dir"], verbose=0) g = memory.cache(f) output = g(1) h = pickle.loads(pickle.dumps(g)) output_dir, _ = g.get_output_dir(1) yield nose.tools.assert_equal, output, h.load_output(output_dir) memory2 = pickle.loads(pickle.dumps(memory)) yield nose.tools.assert_equal, memory.cachedir, memory2.cachedir # Smoke test that pickling a memory with cachedir=None works memory = Memory(cachedir=None, verbose=0) pickle.loads(pickle.dumps(memory)) g = memory.cache(f) gp = pickle.loads(pickle.dumps(g)) gp(1)
def test_memory_numpy(): " Test memory with a function with numpy arrays." # Check with memmapping and without. for mmap_mode in (None, 'r'): accumulator = list() def n(l=None): accumulator.append(1) return l memory = Memory(cachedir=env['dir'], mmap_mode=mmap_mode, verbose=0) memory.clear(warn=False) cached_n = memory.cache(n) rnd = np.random.RandomState(0) for i in range(3): a = rnd.random_sample((10, 10)) for _ in range(3): yield nose.tools.assert_true, np.all(cached_n(a) == a) yield nose.tools.assert_equal, len(accumulator), i + 1
def test_check_call_in_cache(tmpdir): for func in (MemorizedFunc(f, tmpdir.strpath), Memory(location=tmpdir.strpath, verbose=0).cache(f)): result = func.check_call_in_cache(2) assert not result assert isinstance(result, bool) assert func(2) == 5 result = func.check_call_in_cache(2) assert result assert isinstance(result, bool) func.clear()
def test_persistence(tmpdir): # Test the memorized functions can be pickled and restored. memory = Memory(cachedir=tmpdir.strpath, verbose=0) g = memory.cache(f) output = g(1) h = pickle.loads(pickle.dumps(g)) output_dir, _ = h.get_output_dir(1) func_name = _get_func_fullname(f) assert output == _load_output(output_dir, func_name) memory2 = pickle.loads(pickle.dumps(memory)) assert memory.cachedir == memory2.cachedir # Smoke test that pickling a memory with cachedir=None works memory = Memory(cachedir=None, verbose=0) pickle.loads(pickle.dumps(memory)) g = memory.cache(f) gp = pickle.loads(pickle.dumps(g)) gp(1)
def test_memory_numpy_check_mmap_mode(tmpdir): """Check that mmap_mode is respected even at the first call""" memory = Memory(cachedir=tmpdir.strpath, mmap_mode='r', verbose=0) memory.clear(warn=False) @memory.cache() def twice(a): return a * 2 a = np.ones(3) b = twice(a) c = twice(a) assert isinstance(c, np.memmap) assert c.mode == 'r' assert isinstance(b, np.memmap) assert b.mode == 'r'
def test_memory_numpy_check_mmap_mode(): """Check that mmap_mode is respected even at the first call""" memory = Memory(cachedir=env['dir'], mmap_mode='r', verbose=0) memory.clear(warn=False) @memory.cache() def twice(a): return a * 2 a = np.ones(3) b = twice(a) c = twice(a) nose.tools.assert_true(isinstance(c, np.memmap)) nose.tools.assert_equal(c.mode, 'r') nose.tools.assert_true(isinstance(b, np.memmap)) nose.tools.assert_equal(b.mode, 'r')
def test_partial_decoration(tmpdir, ignore, verbose, mmap_mode): "Check cache may be called with kwargs before decorating" memory = Memory(location=tmpdir.strpath, verbose=0) @memory.cache(ignore=ignore, verbose=verbose, mmap_mode=mmap_mode) def z(x): pass assert z.ignore == ignore assert z._verbose == verbose assert z.mmap_mode == mmap_mode
def test_memory_recomputes_after_an_error_why_loading_results( tmpdir, monkeypatch): memory = Memory(location=tmpdir.strpath) def func(arg): # This makes sure that the timestamp returned by two calls of # func are different. This is needed on Windows where # time.time resolution may not be accurate enough time.sleep(0.01) return arg, time.time() cached_func = memory.cache(func) input_arg = 'arg' arg, timestamp = cached_func(input_arg) # Make sure the function is correctly cached assert arg == input_arg # Corrupting output.pkl to make sure that an error happens when # loading the cached result single_cache_item, = memory.store_backend.get_items() output_filename = os.path.join(single_cache_item.path, 'output.pkl') with open(output_filename, 'w') as f: f.write('garbage') recorded_warnings = [] def append_to_record(item): recorded_warnings.append(item) # Make sure that corrupting the file causes recomputation and that # a warning is issued. Need monkeypatch because pytest does not # capture stdlib logging output (see # https://github.com/pytest-dev/pytest/issues/2079) monkeypatch.setattr(cached_func, 'warn', append_to_record) recomputed_arg, recomputed_timestamp = cached_func(arg) assert len(recorded_warnings) == 1 exception_msg = 'Exception while loading results' assert exception_msg in recorded_warnings[0] assert recomputed_arg == arg assert recomputed_timestamp > timestamp
def test_cached_function_race_condition_when_persisting_output_2( tmpdir, capfd): # Test race condition in first attempt at solving # https://github.com/joblib/joblib/issues/490. The race condition # was due to the delay between seeing the cache directory created # (interpreted as the result being cached) and the output.pkl being # pickled. memory = Memory(location=tmpdir.strpath) func_cached = memory.cache(fast_func_with_conditional_complex_output) Parallel(n_jobs=2)(delayed(func_cached)(True if i % 2 == 0 else False) for i in range(3)) stdout, stderr = capfd.readouterr() # Checking both stdout and stderr (ongoing PR #434 may change # logging destination) to make sure there is no exception while # loading the results exception_msg = 'Exception while loading results' assert exception_msg not in stdout assert exception_msg not in stderr
def cached_func(tmpdir_factory): # Create a Memory object to test decorated functions. # We should be careful not to call the decorated functions, so that # cache directories are not created in the temp dir. cachedir = tmpdir_factory.mktemp("joblib_test_func_inspect") mem = Memory(cachedir.strpath) @mem.cache def cached_func_inner(x): return x return cached_func_inner
def test_deprecated_cachedir_behaviour(tmpdir): # verify the right deprecation warnings are raised when using cachedir # option instead of new location parameter. with warns(None) as w: memory = Memory(cachedir=tmpdir.strpath, verbose=0) assert memory.store_backend.location.startswith(tmpdir.strpath) assert len(w) == 1 assert "The 'cachedir' parameter has been deprecated" in str(w[-1].message) with warns(None) as w: memory = Memory() assert memory.cachedir is None assert len(w) == 1 assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message) error_regex = """You set both "location='.+ and "cachedir='.+""" with raises(ValueError, match=error_regex): memory = Memory(location=tmpdir.strpath, cachedir=tmpdir.strpath, verbose=0)
def test_memory_arg_lambda(): " Test memory with a lambda argument." memory = Memory(cachedir=env['dir'], verbose=0) memory.clear(warn=False) accum = {'value': 0} @memory.cache() def run_func(func): accum['value'] += 1 return func() lambda_1 = lambda: 1 lambda_2 = lambda: 2 a = run_func(lambda_1) b = run_func(lambda_1) c = run_func(lambda_2) nose.tools.assert_equal(a, 1) nose.tools.assert_equal(b, 1) nose.tools.assert_equal(c, 2) nose.tools.assert_equal(accum['value'], 2)