def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in e.scheduler.who_has['y'] or
b.address in e.scheduler.who_has['y'])
assert c.who_has['y']
assert e.scheduler.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert c.who_has[x.key]
assert (a.address in e.scheduler.who_has[x.key] or
b.address in e.scheduler.who_has[x.key])
assert e.scheduler.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def test__scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in s.who_has['y'] or
b.address in s.who_has['y'])
assert s.who_has['y']
assert s.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert s.who_has[x.key]
assert (a.address in s.who_has[x.key] or
b.address in s.who_has[x.key])
assert s.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def test_directed_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def test_directed_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
[x] = yield e._scatter([1])
assert e.scheduler.nbytes == {x.key: sizeof(1)}
y = e.submit(inc, x)
yield y._result()
assert e.scheduler.nbytes == {x.key: sizeof(1),
y.key: sizeof(2)}
yield e._shutdown()
def test_nbytes(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._scatter([1])
assert s.nbytes == {x.key: sizeof(1)}
y = e.submit(inc, x)
yield y._result()
assert s.nbytes == {x.key: sizeof(1),
y.key: sizeof(2)}
yield e._shutdown()
def test_remote_scatter_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = yield e._scatter([1, 2, 3])
assert x.key in a.data or x.key in b.data
assert y.key in a.data or y.key in b.data
assert z.key in a.data or z.key in b.data
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def test_nbytes(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._scatter([1])
assert s.nbytes == {x.key: sizeof(1)}
y = e.submit(inc, x)
yield y._result()
assert s.nbytes == {x.key: sizeof(1),
y.key: sizeof(2)}
yield e._shutdown()
def test_remote_scatter_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = yield e._scatter([1, 2, 3])
assert x.key in a.data or x.key in b.data
assert y.key in a.data or y.key in b.data
assert z.key in a.data or z.key in b.data
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def f(s, a, b):
assert s.ncores == {a.address: a.ncores, b.address: b.ncores}
e = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 1)
y = e.submit(inc, 2)
z = e.submit(add, x, y)
result = yield x._result()
assert result == 1 + 1
a, b, c = yield e._scatter([1, 2, 3])
aa, bb, xx = yield e._gather([a, b, x])
assert (aa, bb, xx) == (1, 2, 2)
result = yield e._get({'x': (inc, 1), 'y': (add, 'x', 10)}, 'y')
assert result == 12
def f(s, a, b):
assert s.ncores == {a.address: a.ncores, b.address: b.ncores}
e = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 1)
y = e.submit(inc, 2)
z = e.submit(add, x, y)
result = yield x._result()
assert result == 1 + 1
a, b, c = yield e._scatter([1, 2, 3])
aa, bb, xx = yield e._gather([a, b, x])
assert (aa, bb, xx) == (1, 2, 2)
result = yield e._get({'x': (inc, 1), 'y': (add, 'x', 10)}, 'y')
assert result == 12
def test_garbage_collection_with_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[a] = yield e._scatter([1])
assert a.key in e.futures
assert a.status == 'finished'
assert a.event.is_set()
assert e.refcount[a.key] == 1
a.__del__()
assert e.refcount[a.key] == 0
start = time()
while True:
if a.key not in s.who_has:
break
else:
assert time() < start + 3
yield gen.sleep(0.1)
yield e._shutdown()
def test_garbage_collection_with_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[a] = yield e._scatter([1])
assert a.key in e.futures
assert a.status == 'finished'
assert a.event.is_set()
assert e.refcount[a.key] == 1
a.__del__()
assert e.refcount[a.key] == 0
start = time()
while True:
if a.key not in s.who_has:
break
else:
assert time() < start + 3
yield gen.sleep(0.1)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), delete_batch_time=0, start=False,
loop=loop)
yield e._start()
[a] = yield e._scatter([1])
assert a.key in e.futures
assert a.status == 'finished'
assert a.event.is_set()
assert e.refcount[a.key] == 1
a.__del__()
assert e.refcount[a.key] == 0
start = time()
while True:
if a.key not in c.who_has:
break
else:
assert time() < start + 3
yield gen.sleep(0.1)
yield e._shutdown()
