def test_dont_assume_function_purity(loop):
with cluster() as (s, [a, b]):
with parallel_backend('distributed', loop=loop,
scheduler_host=('127.0.0.1', s['port'])) as p:
x, y = Parallel()(delayed(random2)() for i in range(2))
assert x != y
from joblib.parallel import get_active_backend
ba, _ = get_active_backend()
ba.executor.shutdown()
def check_backend_context_manager(backend_name):
with parallel_backend(backend_name, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert active_n_jobs == 3
assert effective_n_jobs(3) == 3
p = Parallel()
assert p.n_jobs == 3
if backend_name == 'multiprocessing':
assert type(active_backend) == MultiprocessingBackend
assert type(p._backend) == MultiprocessingBackend
elif backend_name == 'threading':
assert type(active_backend) == ThreadingBackend
assert type(p._backend) == ThreadingBackend
elif backend_name.startswith('test_'):
assert type(active_backend) == FakeParallelBackend
assert type(p._backend) == FakeParallelBackend
def check_backend_context_manager(backend_name):
with parallel_backend(backend_name, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert_equal(active_n_jobs, 3)
assert_equal(effective_n_jobs(3), 3)
p = Parallel()
assert_equal(p.n_jobs, 3)
if backend_name == 'multiprocessing':
assert_equal(type(active_backend), MultiprocessingBackend)
assert_equal(type(p._backend), MultiprocessingBackend)
elif backend_name == 'threading':
assert_equal(type(active_backend), ThreadingBackend)
assert_equal(type(p._backend), ThreadingBackend)
elif backend_name.startswith('test_'):
assert_equal(type(active_backend), FakeParallelBackend)
assert_equal(type(p._backend), FakeParallelBackend)
def check_backend_context_manager(backend_name):
with parallel_backend(backend_name, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert_equal(active_n_jobs, 3)
assert_equal(effective_n_jobs(3), 3)
p = Parallel()
assert_equal(p.n_jobs, 3)
if backend_name == "multiprocessing":
assert_equal(type(active_backend), MultiprocessingBackend)
assert_equal(type(p._backend), MultiprocessingBackend)
elif backend_name == "threading":
assert_equal(type(active_backend), ThreadingBackend)
assert_equal(type(p._backend), ThreadingBackend)
elif backend_name.startswith("test_"):
assert_equal(type(active_backend), FakeParallelBackend)
assert_equal(type(p._backend), FakeParallelBackend)
def check_backend_context_manager(backend_name):
with parallel_backend(backend_name, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert active_n_jobs == 3
assert effective_n_jobs(3) == 3
p = Parallel()
assert p.n_jobs == 3
if backend_name == 'multiprocessing':
assert type(active_backend) == MultiprocessingBackend
assert type(p._backend) == MultiprocessingBackend
elif backend_name == 'threading':
assert type(active_backend) == ThreadingBackend
assert type(p._backend) == ThreadingBackend
elif backend_name.startswith('test_'):
assert type(active_backend) == FakeParallelBackend
assert type(p._backend) == FakeParallelBackend
def test_direct_parameterized_backend_context_manager():
assert_equal(_active_backend_type(), MultiprocessingBackend)
# Check that it's possible to pass a backend instance directly,
# without registration
with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
active_backend, active_n_jobs = parallel.get_active_backend()
assert_equal(type(active_backend), ParameterizedParallelBackend)
assert_equal(active_backend.param, 43)
assert_equal(active_n_jobs, 5)
p = Parallel()
assert_equal(p.n_jobs, 5)
assert_true(p._backend is active_backend)
results = p(delayed(sqrt)(i) for i in range(5))
assert_equal(results, [sqrt(i) for i in range(5)])
# The default backend is again retored
assert_equal(_active_backend_type(), MultiprocessingBackend)
def test_direct_parameterized_backend_context_manager():
assert _active_backend_type() == MultiprocessingBackend
# Check that it's possible to pass a backend instance directly,
# without registration
with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
active_backend, active_n_jobs = parallel.get_active_backend()
assert type(active_backend) == ParameterizedParallelBackend
assert active_backend.param == 43
assert active_n_jobs == 5
p = Parallel()
assert p.n_jobs == 5
assert p._backend is active_backend
results = p(delayed(sqrt)(i) for i in range(5))
assert results == [sqrt(i) for i in range(5)]
# The default backend is again retored
assert _active_backend_type() == MultiprocessingBackend
def test_direct_parameterized_backend_context_manager():
assert _active_backend_type() == DefaultBackend
# Check that it's possible to pass a backend instance directly,
# without registration
with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
active_backend, active_n_jobs = parallel.get_active_backend()
assert type(active_backend) == ParameterizedParallelBackend
assert active_backend.param == 43
assert active_n_jobs == 5
p = Parallel()
assert p.n_jobs == 5
assert p._backend is active_backend
results = p(delayed(sqrt)(i) for i in range(5))
assert results == [sqrt(i) for i in range(5)]
# The default backend is again restored
assert _active_backend_type() == DefaultBackend
def test_parameterized_backend_context_manager(monkeypatch):
monkeypatch.setitem(BACKENDS, 'param_backend',
ParameterizedParallelBackend)
assert _active_backend_type() == DefaultBackend
with parallel_backend('param_backend', param=42, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert type(active_backend) == ParameterizedParallelBackend
assert active_backend.param == 42
assert active_n_jobs == 3
p = Parallel()
assert p.n_jobs == 3
assert p._backend is active_backend
results = p(delayed(sqrt)(i) for i in range(5))
assert results == [sqrt(i) for i in range(5)]
# The default backend is again restored
assert _active_backend_type() == DefaultBackend
def test_simple(loop):
with cluster() as (s, [a, b]):
with parallel_backend('distributed', loop=loop,
scheduler_host=('127.0.0.1', s['port'])) as p:
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
from joblib.parallel import get_active_backend
ba, _ = get_active_backend()
ba.executor.shutdown()
def test_direct_parameterized_backend_context_manager():
assert_equal(_active_backend_type(), MultiprocessingBackend)
# Check that it's possible to pass a backend instance directly,
# without registration
with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
active_backend, active_n_jobs = parallel.get_active_backend()
assert_equal(type(active_backend), ParameterizedParallelBackend)
assert_equal(active_backend.param, 43)
assert_equal(active_n_jobs, 5)
p = Parallel()
assert_equal(p.n_jobs, 5)
assert_true(p._backend is active_backend)
results = p(delayed(sqrt)(i) for i in range(5))
assert_equal(results, [sqrt(i) for i in range(5)])
# The default backend is again retored
assert_equal(_active_backend_type(), MultiprocessingBackend)
def test_parameterized_backend_context_manager():
register_parallel_backend("param_backend", ParameterizedParallelBackend)
try:
assert_equal(_active_backend_type(), MultiprocessingBackend)
with parallel_backend("param_backend", param=42, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert_equal(type(active_backend), ParameterizedParallelBackend)
assert_equal(active_backend.param, 42)
assert_equal(active_n_jobs, 3)
p = Parallel()
assert_equal(p.n_jobs, 3)
assert_true(p._backend is active_backend)
results = p(delayed(sqrt)(i) for i in range(5))
assert_equal(results, [sqrt(i) for i in range(5)])
# The default backend is again retored
assert_equal(_active_backend_type(), MultiprocessingBackend)
finally:
del BACKENDS["param_backend"]
def test_parameterized_backend_context_manager():
register_parallel_backend('param_backend', ParameterizedParallelBackend)
try:
assert_equal(_active_backend_type(), MultiprocessingBackend)
with parallel_backend('param_backend', param=42, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert_equal(type(active_backend), ParameterizedParallelBackend)
assert_equal(active_backend.param, 42)
assert_equal(active_n_jobs, 3)
p = Parallel()
assert_equal(p.n_jobs, 3)
assert_true(p._backend is active_backend)
results = p(delayed(sqrt)(i) for i in range(5))
assert_equal(results, [sqrt(i) for i in range(5)])
# The default backend is again retored
assert_equal(_active_backend_type(), MultiprocessingBackend)
finally:
del BACKENDS['param_backend']
def test_parameterized_backend_context_manager():
register_parallel_backend('param_backend', ParameterizedParallelBackend)
try:
assert _active_backend_type() == MultiprocessingBackend
with parallel_backend('param_backend', param=42, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert type(active_backend) == ParameterizedParallelBackend
assert active_backend.param == 42
assert active_n_jobs == 3
p = Parallel()
assert p.n_jobs == 3
assert p._backend is active_backend
results = p(delayed(sqrt)(i) for i in range(5))
assert results == [sqrt(i) for i in range(5)]
# The default backend is again retored
assert _active_backend_type() == MultiprocessingBackend
finally:
del BACKENDS['param_backend']
def test_parameterized_backend_context_manager():
register_parallel_backend('param_backend', ParameterizedParallelBackend)
try:
assert _active_backend_type() == MultiprocessingBackend
with parallel_backend('param_backend', param=42, n_jobs=3):
active_backend, active_n_jobs = parallel.get_active_backend()
assert type(active_backend) == ParameterizedParallelBackend
assert active_backend.param == 42
assert active_n_jobs == 3
p = Parallel()
assert p.n_jobs == 3
assert p._backend is active_backend
results = p(delayed(sqrt)(i) for i in range(5))
assert results == [sqrt(i) for i in range(5)]
# The default backend is again retored
assert _active_backend_type() == MultiprocessingBackend
finally:
del BACKENDS['param_backend']
def _active_backend_type():
return type(parallel.get_active_backend()[0])
def _active_backend_type():
return type(parallel.get_active_backend()[0])