def test_simple(loop):
with cluster() as (s, [a, b]):
with parallel_backend('distributed', loop=loop,
scheduler_host=('127.0.0.1', s['port'])):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
def test_simple(loop):
with cluster() as (s, [a, b]):
with parallel_backend('distributed',
loop=loop,
scheduler_host=('127.0.0.1', s['port'])):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
def test_simple(loop): # noqa: F811
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as client: # noqa: F841
with parallel_backend('dask') as (ba, _):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
def test_simple(loop):
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as client: # noqa: F841
with parallel_backend('dask') as (ba, _):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
def test_simple(loop, joblib):
if joblib is None:
pytest.skip()
Parallel = joblib.Parallel
delayed = joblib.delayed
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as client:
with joblib.parallel_backend('dask') as (ba, _):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
def test_simple(loop):
with cluster() as (s, [a, b]):
with parallel_backend('distributed', loop=loop,
scheduler_host=('127.0.0.1', s['port'])) as p:
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
from joblib.parallel import get_active_backend
ba, _ = get_active_backend()
ba.executor.shutdown()
def test_simple(loop, joblib):
if joblib is None:
pytest.skip()
Parallel = joblib.Parallel
delayed = joblib.delayed
with cluster() as (s, [a, b]):
with joblib.parallel_backend('dask.distributed', loop=loop,
scheduler_host=('127.0.0.1', s['port'])):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
ba, _ = joblib.parallel.get_active_backend()
ba.client.shutdown()
def test_gather_robust_to_nested_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def test_simple(loop, joblib):
if joblib is None:
pytest.skip()
Parallel = joblib.Parallel
delayed = joblib.delayed
with cluster() as (s, [a, b]):
with joblib.parallel_backend('dask.distributed',
loop=loop,
scheduler_host=s['address']) as (ba, _):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
ba.client.close()
def test_gather_robust_to_nested_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def test_simple(loop, joblib):
if joblib is None:
pytest.skip()
Parallel = joblib.Parallel
delayed = joblib.delayed
with cluster() as (s, [a, b]):
with joblib.parallel_backend('dask.distributed',
loop=loop,
scheduler_host=('127.0.0.1', s['port'])):
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
with pytest.raises(ValueError):
Parallel()(delayed(slow_raise_value_error)(i == 3)
for i in range(10))
seq = Parallel()(delayed(inc)(i) for i in range(10))
assert seq == [inc(i) for i in range(10)]
ba, _ = joblib.parallel.get_active_backend()
ba.client.shutdown()
async def f():
results = await All([inc(i) for i in range(10)])
assert results == list(range(1, 11))
start = time()
for tasks in [[throws(), slow()], [slow(), throws()]]:
try:
await All(tasks)
assert False
except ZeroDivisionError:
pass
end = time()
assert end - start < 10
def f():
results = yield All(*[inc(i) for i in range(10)])
assert results == list(range(1, 11))
start = time()
for tasks in [[throws(), slow()], [slow(), throws()]]:
try:
yield All(tasks)
assert False
except ZeroDivisionError:
pass
end = time()
assert end - start < 10
def test_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L1 = e.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = yield L1[0]._result()
assert result == inc(0)
assert len(s.dask) == 5
L2 = e.map(inc, L1)
result = yield L2[1]._result()
assert result == inc(inc(1))
assert len(s.dask) == 10
assert L1[0].key in s.dask[L2[0].key]
total = e.submit(sum, L2)
result = yield total._result()
assert result == sum(map(inc, map(inc, range(5))))
L3 = e.map(add, L1, L2)
result = yield L3[1]._result()
assert result == inc(1) + inc(inc(1))
L4 = e.map(add, range(3), range(4))
results = yield e._gather(L4)
if sys.version_info[0] >= 3:
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = e.map(f, range(5), y=5)
results = yield e._gather(L5)
assert results == list(range(5, 10))
y = e.submit(f, 10)
L6 = e.map(f, range(5), y=y)
results = yield e._gather(L6)
assert results == list(range(20, 25))
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
L1 = e.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = yield L1[0]._result()
assert result == inc(0)
assert len(e.scheduler.dask) == 5
L2 = e.map(inc, L1)
result = yield L2[1]._result()
assert result == inc(inc(1))
assert len(e.scheduler.dask) == 10
assert L1[0].key in e.scheduler.dask[L2[0].key]
total = e.submit(sum, L2)
result = yield total._result()
assert result == sum(map(inc, map(inc, range(5))))
L3 = e.map(add, L1, L2)
result = yield L3[1]._result()
assert result == inc(1) + inc(inc(1))
L4 = e.map(add, range(3), range(4))
results = yield e._gather(L4)
if sys.version_info[0] >= 3:
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = e.map(f, range(5), y=5)
results = yield e._gather(L5)
assert results == list(range(5, 10))
y = e.submit(f, 10)
L6 = e.map(f, range(5), y=y)
results = yield e._gather(L6)
assert results == list(range(20, 25))
yield e._shutdown()
async def test_asyncio_map():
async with AioClient(processes=False) as c:
L1 = c.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = await L1[0]
assert result == inc(0)
L2 = c.map(inc, L1)
result = await L2[1]
assert result == inc(inc(1))
total = c.submit(sum, L2)
result = await total
assert result == sum(map(inc, map(inc, range(5))))
L3 = c.map(add, L1, L2)
result = await L3[1]
assert result == inc(1) + inc(inc(1))
L4 = c.map(add, range(3), range(4))
results = await c.gather(L4)
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = c.map(f, range(5), y=5)
results = await c.gather(L5)
assert results == list(range(5, 10))
y = c.submit(f, 10)
L6 = c.map(f, range(5), y=y)
results = await c.gather(L6)
assert results == list(range(20, 25))
async def test_asyncio_map():
async with AioClient(processes=False) as c:
L1 = c.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, AioFuture) for x in L1)
result = await L1[0]
assert result == inc(0)
L2 = c.map(inc, L1)
result = await L2[1]
assert result == inc(inc(1))
total = c.submit(sum, L2)
result = await total
assert result == sum(map(inc, map(inc, range(5))))
L3 = c.map(add, L1, L2)
result = await L3[1]
assert result == inc(1) + inc(inc(1))
L4 = c.map(add, range(3), range(4))
results = await c.gather(L4)
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = c.map(f, range(5), y=5)
results = await c.gather(L5)
assert results == list(range(5, 10))
y = c.submit(f, 10)
L6 = c.map(f, range(5), y=y)
results = await c.gather(L6)
assert results == list(range(20, 25))
