def test_nested_parallel_warnings():
# The warnings happen in child processes so
# warnings.catch_warnings can not be used for this tests that's
# why we use check_subprocess_call instead
if posix is None:
# This test pass only when fork is the process start method
raise SkipTest('Not a POSIX platform')
template_code = """
import sys
from joblib import Parallel, delayed
def func():
return 42
def parallel_func():
res = Parallel(n_jobs={inner_n_jobs})(delayed(func)() for _ in range(3))
return res
Parallel(n_jobs={outer_n_jobs})(delayed(parallel_func)() for _ in range(5))
"""
# no warnings if inner_n_jobs=1
code = template_code.format(inner_n_jobs=1, outer_n_jobs=2)
check_subprocess_call([sys.executable, '-c', code], stderr_regex='^$')
# warnings if inner_n_jobs != 1
regex = ('Multiprocessing-backed parallel loops cannot ' 'be nested')
code = template_code.format(inner_n_jobs=2, outer_n_jobs=2)
check_subprocess_call([sys.executable, '-c', code], stderr_regex=regex)
def test_parallel_with_interactively_defined_functions(backend):
# When using the "-c" flag, interactive functions defined in __main__
# should work with any backend.
code = CUSTOM_BACKEND_SCRIPT_TEMPLATE.format(backend)
check_subprocess_call([sys.executable, '-c', code],
stdout_regex=r'\[0, 1, 4, 9, 16\]',
timeout=2)
def test_parallel_with_interactively_defined_bound_method(tmpdir):
script = tmpdir.join('joblib_interactive_bound_method_script.py')
script.write(INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENT)
check_subprocess_call([sys.executable, script.strpath],
stdout_regex=r'\[None, None, None\]',
stderr_regex=r'LokyProcess',
timeout=15)
def test_parallel_with_interactively_defined_functions(backend):
# When using the "-c" flag, interactive functions defined in __main__
# should work with any backend.
code = UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAIN.format(backend)
check_subprocess_call([sys.executable, '-c', code],
timeout=10,
stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_nested_parallel_warnings():
# The warnings happen in child processes so
# warnings.catch_warnings can not be used for this tests that's
# why we use check_subprocess_call instead
if posix is None:
# This test pass only when fork is the process start method
raise SkipTest('Not a POSIX platform')
template_code = """
import sys
from joblib import Parallel, delayed
def func():
return 42
def parallel_func():
res = Parallel(n_jobs={inner_n_jobs})(delayed(func)() for _ in range(3))
return res
Parallel(n_jobs={outer_n_jobs})(delayed(parallel_func)() for _ in range(5))
"""
# no warnings if inner_n_jobs=1
code = template_code.format(inner_n_jobs=1, outer_n_jobs=2)
check_subprocess_call([sys.executable, '-c', code],
stderr_regex='^$')
# warnings if inner_n_jobs != 1
regex = ('Multiprocessing-backed parallel loops cannot '
'be nested')
code = template_code.format(inner_n_jobs=2, outer_n_jobs=2)
check_subprocess_call([sys.executable, '-c', code],
stderr_regex=regex)
def test_check_subprocess_call_non_matching_regex():
code = '42'
non_matching_pattern = '_no_way_this_matches_anything_'
with raises(ValueError) as excinfo:
check_subprocess_call([sys.executable, '-c', code],
stdout_regex=non_matching_pattern)
excinfo.match('Unexpected stdout.+{}'.format(non_matching_pattern))
def test_check_subprocess_call_non_matching_regex():
code = '42'
non_matching_pattern = '_no_way_this_matches_anything_'
with raises(ValueError) as excinfo:
check_subprocess_call([sys.executable, '-c', code],
stdout_regex=non_matching_pattern)
excinfo.match('Unexpected stdout.+{}'.format(non_matching_pattern))
def test_parallel_with_unpicklable_functions_in_args(
backend, define_func, callable_position, tmpdir):
code = UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAIN.format(
define_func, backend, callable_position)
code_file = tmpdir.join("unpicklable_func_script.py")
code_file.write(code)
check_subprocess_call(
[sys.executable, code_file.strpath], timeout=10,
stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_no_ressource_tracker_on_import():
code = """if True:
import joblib
from joblib.externals.loky.backend import resource_tracker
# The following line would raise RuntimeError if the
# start_method is already set.
msg = "loky.resource_tracker has been spawned on import"
assert resource_tracker._resource_tracker._fd is None, msg
"""
check_subprocess_call([sys.executable, '-c', code])
def test_parallel_with_interactively_defined_functions_default_backend(tmpdir):
# The default backend (loky) accepts interactive functions defined in
# __main__ and does not require if __name__ == '__main__' even when
# the __main__ module is defined by the result of the execution of a
# filesystem script.
script = tmpdir.join('joblib_default_backend_script.py')
script.write(DEFAULT_BACKEND_SCRIPT_CONTENT)
check_subprocess_call([sys.executable, script.strpath],
stdout_regex=r'\[0, 1, 4, 9, 16\]',
timeout=2)
def test_no_semaphore_tracker_on_import():
# check that importing joblib does not implicitly spawn a ressource tracker
# or a semaphore tracker
code = """if True:
import joblib
from multiprocessing import semaphore_tracker
# The following line would raise RuntimeError if the
# start_method is already set.
msg = "multiprocessing.semaphore_tracker has been spawned on import"
assert semaphore_tracker._semaphore_tracker._fd is None, msg"""
check_subprocess_call([sys.executable, '-c', code])
def test_no_start_method_side_effect_on_import():
# check that importing joblib does not implicitly set the global
# start_method for multiprocessing.
code = """if True:
import joblib
import multiprocessing as mp
# The following line would raise RuntimeError if the
# start_method is already set.
mp.set_start_method("loky")
"""
check_subprocess_call([sys.executable, '-c', code])
def test_check_subprocess_call():
code = '\n'.join(['result = 1 + 2 * 3',
'print(result)',
'my_list = [1, 2, 3]',
'print(my_list)'])
check_subprocess_call([sys.executable, '-c', code])
# Now checking stdout with a regex
check_subprocess_call([sys.executable, '-c', code],
# Regex needed for platform-specific line endings
stdout_regex=r'7\s{1,2}\[1, 2, 3\]')
def test_check_subprocess_call():
code = '\n'.join(['result = 1 + 2 * 3',
'print(result)',
'my_list = [1, 2, 3]',
'print(my_list)'])
check_subprocess_call([sys.executable, '-c', code])
# Now checking stdout with a regex
check_subprocess_call([sys.executable, '-c', code],
# Regex needed for platform-specific line endings
stdout_regex=r'7\s{1,2}\[1, 2, 3\]')
def test_parallel_with_interactively_defined_functions():
# When functions are defined interactively in a python/IPython
# session, we want to be able to use them with joblib.Parallel
if posix is None:
# This test pass only when fork is the process start method
raise SkipTest('Not a POSIX platform')
code = '\n\n'.join([
'from joblib import Parallel, delayed', 'def square(x): return x**2',
'print(Parallel(n_jobs=2)(delayed(square)(i) for i in range(5)))'
])
check_subprocess_call([sys.executable, '-c', code],
stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_check_subprocess_call_non_zero_return_code():
code_with_non_zero_exit = '\n'.join([
'import sys', 'print("writing on stdout")',
'sys.stderr.write("writing on stderr")', 'sys.exit(123)'
])
pattern = re.compile(
'Non-zero return code: 123.+'
'Stdout:\nwriting on stdout.+'
'Stderr:\nwriting on stderr', re.DOTALL)
with raises(ValueError) as excinfo:
check_subprocess_call([sys.executable, '-c', code_with_non_zero_exit])
excinfo.match(pattern)
def test_check_subprocess_call_non_zero_return_code():
code_with_non_zero_exit = '\n'.join([
'import sys',
'print("writing on stdout")',
'sys.stderr.write("writing on stderr")',
'sys.exit(123)'])
pattern = re.compile('Non-zero return code: 123.+'
'Stdout:\nwriting on stdout.+'
'Stderr:\nwriting on stderr', re.DOTALL)
with raises(ValueError) as excinfo:
check_subprocess_call([sys.executable, '-c', code_with_non_zero_exit])
excinfo.match(pattern)
def test_parallel_with_interactively_defined_functions():
# When functions are defined interactively in a python/IPython
# session, we want to be able to use them with joblib.Parallel
if posix is None:
# This test pass only when fork is the process start method
raise SkipTest('Not a POSIX platform')
code = '\n\n'.join([
'from joblib import Parallel, delayed',
'def square(x): return x**2',
'print(Parallel(n_jobs=2)(delayed(square)(i) for i in range(5)))'])
check_subprocess_call([sys.executable, '-c', code],
stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_check_subprocess_call_timeout():
code_timing_out = '\n'.join([
'import time', 'import sys', 'print("before sleep on stdout")',
'sys.stdout.flush()', 'sys.stderr.write("before sleep on stderr")',
'sys.stderr.flush()', 'time.sleep(1.1)',
'print("process should have be killed before")', 'sys.stdout.flush()'
])
pattern = re.compile(
'Non-zero return code:.+'
'Stdout:\nbefore sleep on stdout\s+'
'Stderr:\nbefore sleep on stderr', re.DOTALL)
with raises(ValueError) as excinfo:
check_subprocess_call([sys.executable, '-c', code_timing_out],
timeout=1)
excinfo.match(pattern)
def test_parallel_with_interactively_defined_functions():
# When functions are defined interactively in a python/IPython
# session, we want to be able to use them with joblib.Parallel
try:
import posix
except ImportError:
# This test pass only when fork is the process start method
raise nose.SkipTest("Not a POSIX platform")
code = "\n\n".join(
[
"from joblib import Parallel, delayed",
"def sqrt(x): return x**2",
"print(Parallel(n_jobs=2)(delayed(sqrt)(i) for i in range(5)))",
]
)
check_subprocess_call([sys.executable, "-c", code], stdout_regex=r"\[0, 1, 4, 9, 16\]")
def test_check_subprocess_call_timeout():
code_timing_out = '\n'.join([
'import time',
'import sys',
'print("before sleep on stdout")',
'sys.stdout.flush()',
'sys.stderr.write("before sleep on stderr")',
'sys.stderr.flush()',
'time.sleep(1.1)',
'print("process should have be killed before")',
'sys.stdout.flush()'])
pattern = re.compile('Non-zero return code:.+'
'Stdout:\nbefore sleep on stdout\s+'
'Stderr:\nbefore sleep on stderr',
re.DOTALL)
with raises(ValueError) as excinfo:
check_subprocess_call([sys.executable, '-c', code_timing_out],
timeout=1)
excinfo.match(pattern)
def test_check_subprocess_call_wrong_command():
wrong_command = '_a_command_that_does_not_exist_'
with raises(OSError):
check_subprocess_call([wrong_command])
def test_check_subprocess_call_wrong_command():
wrong_command = '_a_command_that_does_not_exist_'
with raises(OSError):
check_subprocess_call([wrong_command])
