def test__scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in s.who_has['y'] or
b.address in s.who_has['y'])
assert s.who_has['y']
assert s.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert s.who_has[x.key]
assert (a.address in s.who_has[x.key] or
b.address in s.who_has[x.key])
assert s.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in e.scheduler.who_has['y'] or
b.address in e.scheduler.who_has['y'])
assert c.who_has['y']
assert e.scheduler.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert c.who_has[x.key]
assert (a.address in e.scheduler.who_has[x.key] or
b.address in e.scheduler.who_has[x.key])
assert e.scheduler.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list_kwarg, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list_kwarg, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(inc, 10)
y = e.submit(inc, x)
result = yield e._gather(x)
assert result == 11
result = yield e._gather([x])
assert result == [11]
result = yield e._gather({'x': x, 'y': [y]})
assert result == {'x': 11, 'y': [12]}
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(inc, 10)
y = e.submit(inc, x)
result = yield e._gather(x)
assert result == 11
result = yield e._gather([x])
assert result == [11]
result = yield e._gather({'x': x, 'y': [y]})
assert result == {'x': 11, 'y': [12]}
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 10)
y = e.submit(inc, x)
result = yield e._gather(x)
assert result == 11
result = yield e._gather([x])
assert result == [11]
result = yield e._gather({'x': x, 'y': [y]})
assert result == {'x': 11, 'y': [12]}
yield e._shutdown()
def test_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
y = e.submit(inc, x)
result = yield e._gather(x)
assert result == 11
result = yield e._gather([x])
assert result == [11]
result = yield e._gather({'x': x, 'y': [y]})
assert result == {'x': 11, 'y': [12]}
yield e._shutdown()
def test_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
y = e.submit(inc, x)
result = yield e._gather(x)
assert result == 11
result = yield e._gather([x])
assert result == [11]
result = yield e._gather({'x': x, 'y': [y]})
assert result == {'x': 11, 'y': [12]}
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def test_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L1 = e.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = yield L1[0]._result()
assert result == inc(0)
assert len(s.dask) == 5
L2 = e.map(inc, L1)
result = yield L2[1]._result()
assert result == inc(inc(1))
assert len(s.dask) == 10
assert L1[0].key in s.dask[L2[0].key]
total = e.submit(sum, L2)
result = yield total._result()
assert result == sum(map(inc, map(inc, range(5))))
L3 = e.map(add, L1, L2)
result = yield L3[1]._result()
assert result == inc(1) + inc(inc(1))
L4 = e.map(add, range(3), range(4))
results = yield e._gather(L4)
if sys.version_info[0] >= 3:
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = e.map(f, range(5), y=5)
results = yield e._gather(L5)
assert results == list(range(5, 10))
y = e.submit(f, 10)
L6 = e.map(f, range(5), y=y)
results = yield e._gather(L6)
assert results == list(range(20, 25))
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
L1 = e.map(inc, range(5))
assert len(L1) == 5
assert isdistinct(x.key for x in L1)
assert all(isinstance(x, Future) for x in L1)
result = yield L1[0]._result()
assert result == inc(0)
assert len(e.dask) == 5
L2 = e.map(inc, L1)
result = yield L2[1]._result()
assert result == inc(inc(1))
assert len(e.dask) == 10
assert L1[0].key in e.dask[L2[0].key]
total = e.submit(sum, L2)
result = yield total._result()
assert result == sum(map(inc, map(inc, range(5))))
L3 = e.map(add, L1, L2)
result = yield L3[1]._result()
assert result == inc(1) + inc(inc(1))
L4 = e.map(add, range(3), range(4))
results = yield e._gather(L4)
if sys.version_info[0] >= 3:
assert results == list(map(add, range(3), range(4)))
def f(x, y=10):
return x + y
L5 = e.map(f, range(5), y=5)
results = yield e._gather(L5)
assert results == list(range(5, 10))
y = e.submit(f, 10)
L6 = e.map(f, range(5), y=y)
results = yield e._gather(L6)
assert results == list(range(20, 25))
yield e._shutdown()
def test_badly_serialized_input(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
o = BadlySerializedObject()
future = e.submit(inc, o)
futures = e.map(inc, range(10))
L = yield e._gather(futures)
assert list(L) == list(map(inc, range(10)))
def test_map_quotes(s, a, b):
def assert_list(x, z=[]):
return isinstance(x, list) and isinstance(z, list)
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def test_map_quotes(s, a, b):
def assert_list(x, z=[]):
return isinstance(x, list) and isinstance(z, list)
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(assert_list, [[1, 2, 3], [4]])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], z=[10])
result = yield e._gather(L)
assert all(result)
L = e.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
result = yield e._gather(L)
assert all(result)
yield e._shutdown()
def test_nbytes_determines_worker(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(identity, 1, workers=[a.address[0]])
y = e.submit(identity, tuple(range(100)), workers=[b.address[0]])
yield e._gather([x, y])
z = e.submit(lambda x, y: None, x, y)
yield z._result()
assert s.who_has[z.key] == {b.address}
yield e._shutdown()
def test_nbytes_determines_worker(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(identity, 1, workers=[a.address[0]])
y = e.submit(identity, tuple(range(100)), workers=[b.address[0]])
yield e._gather([x, y])
z = e.submit(lambda x, y: None, x, y)
yield z._result()
assert s.who_has[z.key] == {b.address}
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(identity, 1, workers=[a.address[0]])
y = e.submit(identity, tuple(range(100)), workers=[b.address[0]])
yield e._gather([x, y])
z = e.submit(lambda x, y: None, x, y)
yield z._result()
assert e.scheduler.who_has[z.key] == {b.address}
yield e._shutdown()
def test_remote_scatter_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = yield e._scatter([1, 2, 3])
assert x.key in a.data or x.key in b.data
assert y.key in a.data or y.key in b.data
assert z.key in a.data or z.key in b.data
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def test_remote_scatter_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = yield e._scatter([1, 2, 3])
assert x.key in a.data or x.key in b.data
assert y.key in a.data or y.key in b.data
assert z.key in a.data or z.key in b.data
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def test_map_on_futures_with_kwargs(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
def f(x, y=10):
return x + y
futures = e.map(inc, range(10))
futures2 = e.map(f, futures, y=20)
results = yield e._gather(futures2)
assert results == [i + 1 + 20 for i in range(10)]
future = e.submit(inc, 100)
future2 = e.submit(f, future, y=200)
result = yield future2._result()
assert result == 100 + 1 + 200
def test_map_on_futures_with_kwargs(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
def f(x, y=10):
return x + y
futures = e.map(inc, range(10))
futures2 = e.map(f, futures, y=20)
results = yield e._gather(futures2)
assert results == [i + 1 + 20 for i in range(10)]
future = e.submit(inc, 100)
future2 = e.submit(f, future, y=200)
result = yield future2._result()
assert result == 100 + 1 + 200
def test_gather_robust_to_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(s, a, b):
assert s.ncores == {a.address: a.ncores, b.address: b.ncores}
e = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 1)
y = e.submit(inc, 2)
z = e.submit(add, x, y)
result = yield x._result()
assert result == 1 + 1
a, b, c = yield e._scatter([1, 2, 3])
aa, bb, xx = yield e._gather([a, b, x])
assert (aa, bb, xx) == (1, 2, 2)
result = yield e._get({'x': (inc, 1), 'y': (add, 'x', 10)}, 'y')
assert result == 12
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(s, a, b):
assert s.ncores == {a.address: a.ncores, b.address: b.ncores}
e = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 1)
y = e.submit(inc, 2)
z = e.submit(add, x, y)
result = yield x._result()
assert result == 1 + 1
a, b, c = yield e._scatter([1, 2, 3])
aa, bb, xx = yield e._gather([a, b, x])
assert (aa, bb, xx) == (1, 2, 2)
result = yield e._get({'x': (inc, 1), 'y': (add, 'x', 10)}, 'y')
assert result == 12
def test_gather_robust_to_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = e.map(inc, range(3))
yield _wait([x, y, z]) # everything computed
for q in [x, y]:
if q.key in a.data:
del a.data[q.key]
if q.key in b.data:
del b.data[q.key]
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def test_async_compute(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
from dask.imperative import do, value
x = value(1)
y = do(inc)(x)
z = do(dec)(x)
yy, zz, aa = e.compute(y, z, 3, sync=False)
assert isinstance(yy, Future)
assert isinstance(zz, Future)
assert aa == 3
result = yield e._gather([yy, zz])
assert result == [2, 0]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
from dask.imperative import do, value
x = value(1)
y = do(inc)(x)
z = do(dec)(x)
yy, zz, aa = e.compute(y, z, 3, sync=False)
assert isinstance(yy, Future)
assert isinstance(zz, Future)
assert aa == 3
result = yield e._gather([yy, zz])
assert result == [2, 0]
yield e._shutdown()
def test_gather_robust_to_nested_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def test_gather_robust_to_nested_missing_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
w = e.submit(inc, 1)
x = e.submit(inc, w)
y = e.submit(inc, x)
z = e.submit(inc, y)
yield _wait([z])
for worker in [a, b]:
for datum in [y, z]:
if datum.key in worker.data:
del worker.data[datum.key]
result = yield e._gather([z])
assert result == [inc(inc(inc(inc(1))))]
yield e._shutdown()
def test_errors_dont_block():
c = Center('127.0.0.1')
c.listen(0)
w = Worker(c.ip, c.port, ncores=1, ip='127.0.0.1')
e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
yield w._start()
yield e._start()
L = [e.submit(inc, 1),
e.submit(throws, 1),
e.submit(inc, 2),
e.submit(throws, 2)]
start = time()
while not (L[0].status == L[2].status == 'finished'):
assert time() < start + 5
yield gen.sleep(0.01)
result = yield e._gather([L[0], L[2]])
assert result == [2, 3]
yield w._close()
c.stop()
def test_errors_dont_block():
c = Center('127.0.0.1')
c.listen(0)
w = Worker(c.ip, c.port, ncores=1, ip='127.0.0.1')
e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
yield w._start()
yield e._start()
L = [e.submit(inc, 1),
e.submit(throws, 1),
e.submit(inc, 2),
e.submit(throws, 2)]
start = time()
while not (L[0].status == L[2].status == 'finished'):
assert time() < start + 5
yield gen.sleep(0.01)
result = yield e._gather([L[0], L[2]])
assert result == [2, 3]
yield w._close()
c.stop()
