def test_nested_parallel_warnings(): # The warnings happen in child processes so # warnings.catch_warnings can not be used for this tests that's # why we use check_subprocess_call instead if posix is None: # This test pass only when fork is the process start method raise SkipTest('Not a POSIX platform') template_code = """ import sys from joblib import Parallel, delayed def func(): return 42 def parallel_func(): res = Parallel(n_jobs={inner_n_jobs})(delayed(func)() for _ in range(3)) return res Parallel(n_jobs={outer_n_jobs})(delayed(parallel_func)() for _ in range(5)) """ # no warnings if inner_n_jobs=1 code = template_code.format(inner_n_jobs=1, outer_n_jobs=2) check_subprocess_call([sys.executable, '-c', code], stderr_regex='^$') # warnings if inner_n_jobs != 1 regex = ('Multiprocessing-backed parallel loops cannot ' 'be nested') code = template_code.format(inner_n_jobs=2, outer_n_jobs=2) check_subprocess_call([sys.executable, '-c', code], stderr_regex=regex)
def test_parallel_with_interactively_defined_functions(backend): # When using the "-c" flag, interactive functions defined in __main__ # should work with any backend. code = CUSTOM_BACKEND_SCRIPT_TEMPLATE.format(backend) check_subprocess_call([sys.executable, '-c', code], stdout_regex=r'\[0, 1, 4, 9, 16\]', timeout=2)
def test_parallel_with_interactively_defined_bound_method(tmpdir): script = tmpdir.join('joblib_interactive_bound_method_script.py') script.write(INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENT) check_subprocess_call([sys.executable, script.strpath], stdout_regex=r'\[None, None, None\]', stderr_regex=r'LokyProcess', timeout=15)
def test_parallel_with_interactively_defined_functions(backend): # When using the "-c" flag, interactive functions defined in __main__ # should work with any backend. code = UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAIN.format(backend) check_subprocess_call([sys.executable, '-c', code], timeout=10, stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_nested_parallel_warnings(): # The warnings happen in child processes so # warnings.catch_warnings can not be used for this tests that's # why we use check_subprocess_call instead if posix is None: # This test pass only when fork is the process start method raise SkipTest('Not a POSIX platform') template_code = """ import sys from joblib import Parallel, delayed def func(): return 42 def parallel_func(): res = Parallel(n_jobs={inner_n_jobs})(delayed(func)() for _ in range(3)) return res Parallel(n_jobs={outer_n_jobs})(delayed(parallel_func)() for _ in range(5)) """ # no warnings if inner_n_jobs=1 code = template_code.format(inner_n_jobs=1, outer_n_jobs=2) check_subprocess_call([sys.executable, '-c', code], stderr_regex='^$') # warnings if inner_n_jobs != 1 regex = ('Multiprocessing-backed parallel loops cannot ' 'be nested') code = template_code.format(inner_n_jobs=2, outer_n_jobs=2) check_subprocess_call([sys.executable, '-c', code], stderr_regex=regex)
def test_check_subprocess_call_non_matching_regex(): code = '42' non_matching_pattern = '_no_way_this_matches_anything_' with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code], stdout_regex=non_matching_pattern) excinfo.match('Unexpected stdout.+{}'.format(non_matching_pattern))
def test_check_subprocess_call_non_matching_regex(): code = '42' non_matching_pattern = '_no_way_this_matches_anything_' with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code], stdout_regex=non_matching_pattern) excinfo.match('Unexpected stdout.+{}'.format(non_matching_pattern))
def test_parallel_with_unpicklable_functions_in_args( backend, define_func, callable_position, tmpdir): code = UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAIN.format( define_func, backend, callable_position) code_file = tmpdir.join("unpicklable_func_script.py") code_file.write(code) check_subprocess_call( [sys.executable, code_file.strpath], timeout=10, stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_no_ressource_tracker_on_import(): code = """if True: import joblib from joblib.externals.loky.backend import resource_tracker # The following line would raise RuntimeError if the # start_method is already set. msg = "loky.resource_tracker has been spawned on import" assert resource_tracker._resource_tracker._fd is None, msg """ check_subprocess_call([sys.executable, '-c', code])
def test_parallel_with_interactively_defined_functions_default_backend(tmpdir): # The default backend (loky) accepts interactive functions defined in # __main__ and does not require if __name__ == '__main__' even when # the __main__ module is defined by the result of the execution of a # filesystem script. script = tmpdir.join('joblib_default_backend_script.py') script.write(DEFAULT_BACKEND_SCRIPT_CONTENT) check_subprocess_call([sys.executable, script.strpath], stdout_regex=r'\[0, 1, 4, 9, 16\]', timeout=2)
def test_no_semaphore_tracker_on_import(): # check that importing joblib does not implicitly spawn a ressource tracker # or a semaphore tracker code = """if True: import joblib from multiprocessing import semaphore_tracker # The following line would raise RuntimeError if the # start_method is already set. msg = "multiprocessing.semaphore_tracker has been spawned on import" assert semaphore_tracker._semaphore_tracker._fd is None, msg""" check_subprocess_call([sys.executable, '-c', code])
def test_no_start_method_side_effect_on_import(): # check that importing joblib does not implicitly set the global # start_method for multiprocessing. code = """if True: import joblib import multiprocessing as mp # The following line would raise RuntimeError if the # start_method is already set. mp.set_start_method("loky") """ check_subprocess_call([sys.executable, '-c', code])
def test_check_subprocess_call(): code = '\n'.join(['result = 1 + 2 * 3', 'print(result)', 'my_list = [1, 2, 3]', 'print(my_list)']) check_subprocess_call([sys.executable, '-c', code]) # Now checking stdout with a regex check_subprocess_call([sys.executable, '-c', code], # Regex needed for platform-specific line endings stdout_regex=r'7\s{1,2}\[1, 2, 3\]')
def test_check_subprocess_call(): code = '\n'.join(['result = 1 + 2 * 3', 'print(result)', 'my_list = [1, 2, 3]', 'print(my_list)']) check_subprocess_call([sys.executable, '-c', code]) # Now checking stdout with a regex check_subprocess_call([sys.executable, '-c', code], # Regex needed for platform-specific line endings stdout_regex=r'7\s{1,2}\[1, 2, 3\]')
def test_parallel_with_interactively_defined_functions(): # When functions are defined interactively in a python/IPython # session, we want to be able to use them with joblib.Parallel if posix is None: # This test pass only when fork is the process start method raise SkipTest('Not a POSIX platform') code = '\n\n'.join([ 'from joblib import Parallel, delayed', 'def square(x): return x**2', 'print(Parallel(n_jobs=2)(delayed(square)(i) for i in range(5)))' ]) check_subprocess_call([sys.executable, '-c', code], stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_check_subprocess_call_non_zero_return_code(): code_with_non_zero_exit = '\n'.join([ 'import sys', 'print("writing on stdout")', 'sys.stderr.write("writing on stderr")', 'sys.exit(123)' ]) pattern = re.compile( 'Non-zero return code: 123.+' 'Stdout:\nwriting on stdout.+' 'Stderr:\nwriting on stderr', re.DOTALL) with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code_with_non_zero_exit]) excinfo.match(pattern)
def test_check_subprocess_call_non_zero_return_code(): code_with_non_zero_exit = '\n'.join([ 'import sys', 'print("writing on stdout")', 'sys.stderr.write("writing on stderr")', 'sys.exit(123)']) pattern = re.compile('Non-zero return code: 123.+' 'Stdout:\nwriting on stdout.+' 'Stderr:\nwriting on stderr', re.DOTALL) with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code_with_non_zero_exit]) excinfo.match(pattern)
def test_parallel_with_interactively_defined_functions(): # When functions are defined interactively in a python/IPython # session, we want to be able to use them with joblib.Parallel if posix is None: # This test pass only when fork is the process start method raise SkipTest('Not a POSIX platform') code = '\n\n'.join([ 'from joblib import Parallel, delayed', 'def square(x): return x**2', 'print(Parallel(n_jobs=2)(delayed(square)(i) for i in range(5)))']) check_subprocess_call([sys.executable, '-c', code], stdout_regex=r'\[0, 1, 4, 9, 16\]')
def test_check_subprocess_call_timeout(): code_timing_out = '\n'.join([ 'import time', 'import sys', 'print("before sleep on stdout")', 'sys.stdout.flush()', 'sys.stderr.write("before sleep on stderr")', 'sys.stderr.flush()', 'time.sleep(1.1)', 'print("process should have be killed before")', 'sys.stdout.flush()' ]) pattern = re.compile( 'Non-zero return code:.+' 'Stdout:\nbefore sleep on stdout\s+' 'Stderr:\nbefore sleep on stderr', re.DOTALL) with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code_timing_out], timeout=1) excinfo.match(pattern)
def test_parallel_with_interactively_defined_functions(): # When functions are defined interactively in a python/IPython # session, we want to be able to use them with joblib.Parallel try: import posix except ImportError: # This test pass only when fork is the process start method raise nose.SkipTest("Not a POSIX platform") code = "\n\n".join( [ "from joblib import Parallel, delayed", "def sqrt(x): return x**2", "print(Parallel(n_jobs=2)(delayed(sqrt)(i) for i in range(5)))", ] ) check_subprocess_call([sys.executable, "-c", code], stdout_regex=r"\[0, 1, 4, 9, 16\]")
def test_check_subprocess_call_timeout(): code_timing_out = '\n'.join([ 'import time', 'import sys', 'print("before sleep on stdout")', 'sys.stdout.flush()', 'sys.stderr.write("before sleep on stderr")', 'sys.stderr.flush()', 'time.sleep(1.1)', 'print("process should have be killed before")', 'sys.stdout.flush()']) pattern = re.compile('Non-zero return code:.+' 'Stdout:\nbefore sleep on stdout\s+' 'Stderr:\nbefore sleep on stderr', re.DOTALL) with raises(ValueError) as excinfo: check_subprocess_call([sys.executable, '-c', code_timing_out], timeout=1) excinfo.match(pattern)
def test_check_subprocess_call_wrong_command(): wrong_command = '_a_command_that_does_not_exist_' with raises(OSError): check_subprocess_call([wrong_command])
def test_check_subprocess_call_wrong_command(): wrong_command = '_a_command_that_does_not_exist_' with raises(OSError): check_subprocess_call([wrong_command])