Exemple #1
0
def test_dont_assume_function_purity(loop):
    with cluster() as (s, [a, b]):
        with parallel_backend('distributed', loop=loop,
                scheduler_host=('127.0.0.1', s['port'])) as p:

            x, y = Parallel()(delayed(random2)() for i in range(2))
            assert x != y

            from joblib.parallel import get_active_backend
            ba, _ = get_active_backend()
            ba.executor.shutdown()
Exemple #2
0
def check_backend_context_manager(backend_name):
    with parallel_backend(backend_name, n_jobs=3):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert active_n_jobs == 3
        assert effective_n_jobs(3) == 3
        p = Parallel()
        assert p.n_jobs == 3
        if backend_name == 'multiprocessing':
            assert type(active_backend) == MultiprocessingBackend
            assert type(p._backend) == MultiprocessingBackend
        elif backend_name == 'threading':
            assert type(active_backend) == ThreadingBackend
            assert type(p._backend) == ThreadingBackend
        elif backend_name.startswith('test_'):
            assert type(active_backend) == FakeParallelBackend
            assert type(p._backend) == FakeParallelBackend
def check_backend_context_manager(backend_name):
    with parallel_backend(backend_name, n_jobs=3):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert_equal(active_n_jobs, 3)
        assert_equal(effective_n_jobs(3), 3)
        p = Parallel()
        assert_equal(p.n_jobs, 3)
        if backend_name == 'multiprocessing':
            assert_equal(type(active_backend), MultiprocessingBackend)
            assert_equal(type(p._backend), MultiprocessingBackend)
        elif backend_name == 'threading':
            assert_equal(type(active_backend), ThreadingBackend)
            assert_equal(type(p._backend), ThreadingBackend)
        elif backend_name.startswith('test_'):
            assert_equal(type(active_backend), FakeParallelBackend)
            assert_equal(type(p._backend), FakeParallelBackend)
Exemple #4
0
def check_backend_context_manager(backend_name):
    with parallel_backend(backend_name, n_jobs=3):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert_equal(active_n_jobs, 3)
        assert_equal(effective_n_jobs(3), 3)
        p = Parallel()
        assert_equal(p.n_jobs, 3)
        if backend_name == "multiprocessing":
            assert_equal(type(active_backend), MultiprocessingBackend)
            assert_equal(type(p._backend), MultiprocessingBackend)
        elif backend_name == "threading":
            assert_equal(type(active_backend), ThreadingBackend)
            assert_equal(type(p._backend), ThreadingBackend)
        elif backend_name.startswith("test_"):
            assert_equal(type(active_backend), FakeParallelBackend)
            assert_equal(type(p._backend), FakeParallelBackend)
Exemple #5
0
def check_backend_context_manager(backend_name):
    with parallel_backend(backend_name, n_jobs=3):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert active_n_jobs == 3
        assert effective_n_jobs(3) == 3
        p = Parallel()
        assert p.n_jobs == 3
        if backend_name == 'multiprocessing':
            assert type(active_backend) == MultiprocessingBackend
            assert type(p._backend) == MultiprocessingBackend
        elif backend_name == 'threading':
            assert type(active_backend) == ThreadingBackend
            assert type(p._backend) == ThreadingBackend
        elif backend_name.startswith('test_'):
            assert type(active_backend) == FakeParallelBackend
            assert type(p._backend) == FakeParallelBackend
def test_direct_parameterized_backend_context_manager():
    assert_equal(_active_backend_type(), MultiprocessingBackend)

    # Check that it's possible to pass a backend instance directly,
    # without registration
    with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert_equal(type(active_backend), ParameterizedParallelBackend)
        assert_equal(active_backend.param, 43)
        assert_equal(active_n_jobs, 5)
        p = Parallel()
        assert_equal(p.n_jobs, 5)
        assert_true(p._backend is active_backend)
        results = p(delayed(sqrt)(i) for i in range(5))
    assert_equal(results, [sqrt(i) for i in range(5)])

    # The default backend is again retored
    assert_equal(_active_backend_type(), MultiprocessingBackend)
Exemple #7
0
def test_direct_parameterized_backend_context_manager():
    assert _active_backend_type() == MultiprocessingBackend

    # Check that it's possible to pass a backend instance directly,
    # without registration
    with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert type(active_backend) == ParameterizedParallelBackend
        assert active_backend.param == 43
        assert active_n_jobs == 5
        p = Parallel()
        assert p.n_jobs == 5
        assert p._backend is active_backend
        results = p(delayed(sqrt)(i) for i in range(5))
    assert results == [sqrt(i) for i in range(5)]

    # The default backend is again retored
    assert _active_backend_type() == MultiprocessingBackend
Exemple #8
0
def test_direct_parameterized_backend_context_manager():
    assert _active_backend_type() == DefaultBackend

    # Check that it's possible to pass a backend instance directly,
    # without registration
    with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert type(active_backend) == ParameterizedParallelBackend
        assert active_backend.param == 43
        assert active_n_jobs == 5
        p = Parallel()
        assert p.n_jobs == 5
        assert p._backend is active_backend
        results = p(delayed(sqrt)(i) for i in range(5))
    assert results == [sqrt(i) for i in range(5)]

    # The default backend is again restored
    assert _active_backend_type() == DefaultBackend
Exemple #9
0
def test_parameterized_backend_context_manager(monkeypatch):
    monkeypatch.setitem(BACKENDS, 'param_backend',
                        ParameterizedParallelBackend)
    assert _active_backend_type() == DefaultBackend

    with parallel_backend('param_backend', param=42, n_jobs=3):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert type(active_backend) == ParameterizedParallelBackend
        assert active_backend.param == 42
        assert active_n_jobs == 3
        p = Parallel()
        assert p.n_jobs == 3
        assert p._backend is active_backend
        results = p(delayed(sqrt)(i) for i in range(5))
    assert results == [sqrt(i) for i in range(5)]

    # The default backend is again restored
    assert _active_backend_type() == DefaultBackend
Exemple #10
0
def test_simple(loop):
    with cluster() as (s, [a, b]):
        with parallel_backend('distributed', loop=loop,
                scheduler_host=('127.0.0.1', s['port'])) as p:

            seq = Parallel()(delayed(inc)(i) for i in range(10))
            assert seq == [inc(i) for i in range(10)]

            with pytest.raises(ValueError):
                Parallel()(delayed(slow_raise_value_error)(i == 3)
                    for i in range(10))

            seq = Parallel()(delayed(inc)(i) for i in range(10))
            assert seq == [inc(i) for i in range(10)]

            from joblib.parallel import get_active_backend
            ba, _ = get_active_backend()
            ba.executor.shutdown()
Exemple #11
0
def test_direct_parameterized_backend_context_manager():
    assert_equal(_active_backend_type(), MultiprocessingBackend)

    # Check that it's possible to pass a backend instance directly,
    # without registration
    with parallel_backend(ParameterizedParallelBackend(param=43), n_jobs=5):
        active_backend, active_n_jobs = parallel.get_active_backend()
        assert_equal(type(active_backend), ParameterizedParallelBackend)
        assert_equal(active_backend.param, 43)
        assert_equal(active_n_jobs, 5)
        p = Parallel()
        assert_equal(p.n_jobs, 5)
        assert_true(p._backend is active_backend)
        results = p(delayed(sqrt)(i) for i in range(5))
    assert_equal(results, [sqrt(i) for i in range(5)])

    # The default backend is again retored
    assert_equal(_active_backend_type(), MultiprocessingBackend)
Exemple #12
0
def test_parameterized_backend_context_manager():
    register_parallel_backend("param_backend", ParameterizedParallelBackend)
    try:
        assert_equal(_active_backend_type(), MultiprocessingBackend)

        with parallel_backend("param_backend", param=42, n_jobs=3):
            active_backend, active_n_jobs = parallel.get_active_backend()
            assert_equal(type(active_backend), ParameterizedParallelBackend)
            assert_equal(active_backend.param, 42)
            assert_equal(active_n_jobs, 3)
            p = Parallel()
            assert_equal(p.n_jobs, 3)
            assert_true(p._backend is active_backend)
            results = p(delayed(sqrt)(i) for i in range(5))
        assert_equal(results, [sqrt(i) for i in range(5)])

        # The default backend is again retored
        assert_equal(_active_backend_type(), MultiprocessingBackend)
    finally:
        del BACKENDS["param_backend"]
def test_parameterized_backend_context_manager():
    register_parallel_backend('param_backend', ParameterizedParallelBackend)
    try:
        assert_equal(_active_backend_type(), MultiprocessingBackend)

        with parallel_backend('param_backend', param=42, n_jobs=3):
            active_backend, active_n_jobs = parallel.get_active_backend()
            assert_equal(type(active_backend), ParameterizedParallelBackend)
            assert_equal(active_backend.param, 42)
            assert_equal(active_n_jobs, 3)
            p = Parallel()
            assert_equal(p.n_jobs, 3)
            assert_true(p._backend is active_backend)
            results = p(delayed(sqrt)(i) for i in range(5))
        assert_equal(results, [sqrt(i) for i in range(5)])

        # The default backend is again retored
        assert_equal(_active_backend_type(), MultiprocessingBackend)
    finally:
        del BACKENDS['param_backend']
Exemple #14
0
def test_parameterized_backend_context_manager():
    register_parallel_backend('param_backend', ParameterizedParallelBackend)
    try:
        assert _active_backend_type() == MultiprocessingBackend

        with parallel_backend('param_backend', param=42, n_jobs=3):
            active_backend, active_n_jobs = parallel.get_active_backend()
            assert type(active_backend) == ParameterizedParallelBackend
            assert active_backend.param == 42
            assert active_n_jobs == 3
            p = Parallel()
            assert p.n_jobs == 3
            assert p._backend is active_backend
            results = p(delayed(sqrt)(i) for i in range(5))
        assert results == [sqrt(i) for i in range(5)]

        # The default backend is again retored
        assert _active_backend_type() == MultiprocessingBackend
    finally:
        del BACKENDS['param_backend']
Exemple #15
0
def test_parameterized_backend_context_manager():
    register_parallel_backend('param_backend', ParameterizedParallelBackend)
    try:
        assert _active_backend_type() == MultiprocessingBackend

        with parallel_backend('param_backend', param=42, n_jobs=3):
            active_backend, active_n_jobs = parallel.get_active_backend()
            assert type(active_backend) == ParameterizedParallelBackend
            assert active_backend.param == 42
            assert active_n_jobs == 3
            p = Parallel()
            assert p.n_jobs == 3
            assert p._backend is active_backend
            results = p(delayed(sqrt)(i) for i in range(5))
        assert results == [sqrt(i) for i in range(5)]

        # The default backend is again retored
        assert _active_backend_type() == MultiprocessingBackend
    finally:
        del BACKENDS['param_backend']
Exemple #16
0
def _active_backend_type():
    return type(parallel.get_active_backend()[0])
Exemple #17
0
def _active_backend_type():
    return type(parallel.get_active_backend()[0])