def test_func_dir(tmpdir):
# Test the creation of the memory cache directory for the function.
memory = Memory(location=tmpdir.strpath, verbose=0)
path = __name__.split('.')
path.append('f')
path = tmpdir.join('joblib', *path).strpath
g = memory.cache(f)
# Test that the function directory is created on demand
func_id = _build_func_identifier(f)
location = os.path.join(g.store_backend.location, func_id)
assert location == path
assert os.path.exists(path)
assert memory.location == os.path.dirname(g.store_backend.location)
with warns(DeprecationWarning) as w:
assert memory.cachedir == g.store_backend.location
assert len(w) == 1
assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message)
# Test that the code is stored.
# For the following test to be robust to previous execution, we clear
# the in-memory store
_FUNCTION_HASHES.clear()
assert not g._check_previous_func_code()
assert os.path.exists(os.path.join(path, 'func_code.py'))
assert g._check_previous_func_code()
# Test the robustness to failure of loading previous results.
func_id, args_id = g._get_output_identifiers(1)
output_dir = os.path.join(g.store_backend.location, func_id, args_id)
a = g(1)
assert os.path.exists(output_dir)
os.remove(os.path.join(output_dir, 'output.pkl'))
assert a == g(1)
def test_memory_name_collision(tmpdir):
" Check that name collisions with functions will raise warnings"
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
@memory.cache
def name_collision(x):
""" A first function called name_collision
"""
return x
a = name_collision
@memory.cache
def name_collision(x):
""" A second function called name_collision
"""
return x
b = name_collision
with warns(JobLibCollisionWarning) as warninfo:
a(1)
b(1)
assert len(warninfo) == 1
assert "collision" in str(warninfo[0].message)
def test_call_and_shelve_argument_hash(tmpdir):
# Verify that a warning is raised when accessing arguments_hash
# attribute from MemorizedResult
func = Memory(location=tmpdir.strpath, verbose=0).cache(f)
result = func.call_and_shelve(2)
assert isinstance(result, MemorizedResult)
with warns(DeprecationWarning) as w:
assert result.argument_hash == result.args_id
assert len(w) == 1
assert "The 'argument_hash' attribute has been deprecated" \
in str(w[-1].message)
def test_warning_on_unknown_location_type():
class NonSupportedLocationClass:
pass
unsupported_location = NonSupportedLocationClass()
with warns(UserWarning) as warninfo:
_store_backend_factory("local", location=unsupported_location)
expected_mesage = ("Instanciating a backend using a "
"NonSupportedLocationClass as a location is not "
"supported by joblib")
assert expected_mesage in str(warninfo[0].message)
def test_deprecated_cachedir_behaviour(tmpdir):
# verify the right deprecation warnings are raised when using cachedir
# option instead of new location parameter.
with warns(None) as w:
memory = Memory(cachedir=tmpdir.strpath, verbose=0)
assert memory.store_backend.location.startswith(tmpdir.strpath)
assert len(w) == 1
assert "The 'cachedir' parameter has been deprecated" in str(w[-1].message)
with warns(None) as w:
memory = Memory()
assert memory.cachedir is None
assert len(w) == 1
assert "The 'cachedir' attribute has been deprecated" in str(w[-1].message)
error_regex = """You set both "location='.+ and "cachedir='.+"""
with raises(ValueError, match=error_regex):
memory = Memory(location=tmpdir.strpath, cachedir=tmpdir.strpath,
verbose=0)
def test_memory_in_memory_function_code_change(tmpdir):
_function_to_cache.__code__ = _sum.__code__
mem = Memory(cachedir=tmpdir.strpath, verbose=0)
f = mem.cache(_function_to_cache)
assert f(1, 2) == 3
assert f(1, 2) == 3
with warns(JobLibCollisionWarning):
# Check that inline function modification triggers a cache invalidation
_function_to_cache.__code__ = _product.__code__
assert f(1, 2) == 2
assert f(1, 2) == 2
def test_file_handle_persistence_in_memory_mmap():
obj = np.random.random((10, 10))
buf = io.BytesIO()
numpy_pickle.dump(obj, buf)
with warns(UserWarning) as warninfo:
numpy_pickle.load(buf, mmap_mode='r+')
assert len(warninfo) == 1
assert (str(warninfo[0].message) ==
'In memory persistence is not compatible with mmap_mode '
'"%(mmap_mode)s" flag passed. mmap_mode option will be '
'ignored.' % {
'mmap_mode': 'r+'
})
def test_delayed_check_pickle_deprecated():
class UnpicklableCallable(object):
def __call__(self, *args, **kwargs):
return 42
def __reduce__(self):
raise ValueError()
with warns(DeprecationWarning):
f, args, kwargs = delayed(lambda x: 42, check_pickle=False)('a')
assert f('a') == 42
assert args == ('a', )
assert kwargs == dict()
with warns(DeprecationWarning):
f, args, kwargs = delayed(UnpicklableCallable(),
check_pickle=False)('a', option='b')
assert f('a', option='b') == 42
assert args == ('a', )
assert kwargs == dict(option='b')
with warns(DeprecationWarning):
with raises(ValueError):
delayed(UnpicklableCallable(), check_pickle=True)
def test_compress_mmap_mode_warning(tmpdir):
# Test the warning in case of compress + mmap_mode
rnd = np.random.RandomState(0)
a = rnd.random_sample(10)
this_filename = tmpdir.join('test.pkl').strpath
numpy_pickle.dump(a, this_filename, compress=1)
with warns(UserWarning) as warninfo:
numpy_pickle.load(this_filename, mmap_mode='r+')
assert len(warninfo) == 1
assert (str(warninfo[0].message) ==
'mmap_mode "%(mmap_mode)s" is not compatible with compressed '
'file %(filename)s. "%(mmap_mode)s" flag will be ignored.' % {
'filename': this_filename,
'mmap_mode': 'r+'
})
def test_memory_warning_collision_detection(tmpdir):
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(location=tmpdir.strpath, verbose=0)
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
with warns(JobLibCollisionWarning) as warninfo:
a1(1)
b1(1)
a1(0)
assert len(warninfo) == 2
assert "cannot detect" in str(warninfo[0].message).lower()
def test_memory_warning_lambda_collisions(tmpdir):
# Check that multiple use of lambda will raise collisions
memory = Memory(location=tmpdir.strpath, verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warns(JobLibCollisionWarning) as warninfo:
assert a(0) == 0
assert b(1) == 2
assert a(1) == 1
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
assert len(warninfo) == 4
def test_memory_warning_collision_detection(tmpdir):
# Check that collisions impossible to detect will raise appropriate
# warnings.
memory = Memory(location=tmpdir.strpath, verbose=0)
a1 = eval('lambda x: x')
a1 = memory.cache(a1)
b1 = eval('lambda x: x+1')
b1 = memory.cache(b1)
with warns(JobLibCollisionWarning) as warninfo:
a1(1)
b1(1)
a1(0)
assert len(warninfo) == 2
assert "cannot detect" in str(warninfo[0].message).lower()
def test_memory_warning_lambda_collisions(tmpdir):
# Check that multiple use of lambda will raise collisions
memory = Memory(location=tmpdir.strpath, verbose=0)
a = lambda x: x
a = memory.cache(a)
b = lambda x: x + 1
b = memory.cache(b)
with warns(JobLibCollisionWarning) as warninfo:
assert a(0) == 0
assert b(1) == 2
assert a(1) == 1
# In recent Python versions, we can retrieve the code of lambdas,
# thus nothing is raised
assert len(warninfo) == 4
def test_cache_size_warning(tmpdir, cache_size):
# Check deprecation warning raised when cache size is not None
filename = tmpdir.join('test.pkl').strpath
rnd = np.random.RandomState(0)
a = rnd.random_sample((10, 2))
warnings.simplefilter("always")
with warns(None) as warninfo:
numpy_pickle.dump(a, filename, cache_size=cache_size)
expected_nb_warnings = 1 if cache_size is not None else 0
assert len(warninfo) == expected_nb_warnings
for w in warninfo:
assert w.category == DeprecationWarning
assert (str(w.message) ==
"Please do not set 'cache_size' in joblib.dump, this "
"parameter has no effect and will be removed. You "
"used 'cache_size={0}'".format(cache_size))
def test_file_handle_persistence_compressed_mmap(tmpdir):
obj = np.random.random((10, 10))
filename = tmpdir.join('test.pkl').strpath
with open(filename, 'wb') as f:
numpy_pickle.dump(obj, f, compress=('gzip', 3))
with closing(gzip.GzipFile(filename, 'rb')) as f:
with warns(UserWarning) as warninfo:
numpy_pickle.load(f, mmap_mode='r+')
assert len(warninfo) == 1
assert (str(warninfo[0].message) ==
'"%(fileobj)r" is not a raw file, mmap_mode "%(mmap_mode)s" '
'flag will be ignored.' % {
'fileobj': f,
'mmap_mode': 'r+'
})
def test_main_thread_renamed_no_warning(backend, monkeypatch):
# Check that no default backend relies on the name of the main thread:
# https://github.com/joblib/joblib/issues/180#issuecomment-253266247
# Some programs use a different name for the main thread. This is the case
# for uWSGI apps for instance.
monkeypatch.setattr(target=threading.current_thread(),
name='name',
value='some_new_name_for_the_main_thread')
with warns(None) as warninfo:
results = Parallel(n_jobs=2, backend=backend)(delayed(square)(x)
for x in range(3))
assert results == [0, 1, 4]
# The multiprocessing backend will raise a warning when detecting that is
# started from the non-main thread. Let's check that there is no false
# positive because of the name change.
assert len(warninfo) == 0
def _check_pickle(filename, expected_list):
"""Helper function to test joblib pickle content.
Note: currently only pickles containing an iterable are supported
by this function.
"""
version_match = re.match(r'.+py(\d)(\d).+', filename)
py_version_used_for_writing = int(version_match.group(1))
py_version_to_default_pickle_protocol = {2: 2, 3: 3}
pickle_reading_protocol = py_version_to_default_pickle_protocol.get(3, 4)
pickle_writing_protocol = py_version_to_default_pickle_protocol.get(
py_version_used_for_writing, 4)
if pickle_reading_protocol >= pickle_writing_protocol:
try:
with warns(None) as warninfo:
warnings.simplefilter('always')
warnings.filterwarnings(
'ignore',
module='numpy',
message='The compiler package is deprecated')
result_list = numpy_pickle.load(filename)
filename_base = os.path.basename(filename)
expected_nb_warnings = 1 if ("_0.9" in filename_base
or "_0.8.4" in filename_base) else 0
assert len(warninfo) == expected_nb_warnings
for w in warninfo:
assert w.category == DeprecationWarning
assert (str(w.message) ==
"The file '{0}' has been generated with a joblib "
"version less than 0.10. Please regenerate this "
"pickle file.".format(filename))
for result, expected in zip(result_list, expected_list):
if isinstance(expected, np.ndarray):
assert result.dtype == expected.dtype
np.testing.assert_equal(result, expected)
else:
assert result == expected
except Exception as exc:
# When trying to read with python 3 a pickle generated
# with python 2 we expect a user-friendly error
if py_version_used_for_writing == 2:
assert isinstance(exc, ValueError)
message = ('You may be trying to read with '
'python 3 a joblib pickle generated with python 2.')
assert message in str(exc)
elif filename.endswith('.lz4') and with_lz4.args[0]:
assert isinstance(exc, ValueError)
assert LZ4_NOT_INSTALLED_ERROR in str(exc)
else:
raise
else:
# Pickle protocol used for writing is too high. We expect a
# "unsupported pickle protocol" error message
try:
numpy_pickle.load(filename)
raise AssertionError('Numpy pickle loading should '
'have raised a ValueError exception')
except ValueError as e:
message = 'unsupported pickle protocol: {0}'.format(
pickle_writing_protocol)
assert message in str(e.args)