def f(c, a, b):
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
assert isinstance(e1.center, rpc)
assert isinstance(e1.scheduler, Scheduler)
s = Scheduler((c.ip, c.port))
yield s.sync_center()
done = s.start()
e2 = Executor(s, start=False, loop=loop)
yield e2._start()
assert isinstance(e2.center, rpc)
assert isinstance(e2.scheduler, Scheduler)
s.listen(8042)
e3 = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e3._start()
assert isinstance(e3.center, rpc)
assert isinstance(e3.scheduler, rpc)
s.stop()
yield e1._shutdown()
yield e2._shutdown()
yield e3._shutdown()
def f(c, a, b):
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
assert isinstance(e1.center, rpc)
assert isinstance(e1.scheduler, Scheduler)
s = Scheduler((c.ip, c.port))
yield s.sync_center()
done = s.start()
e2 = Executor(s, start=False, loop=loop)
yield e2._start()
assert isinstance(e2.center, rpc)
assert isinstance(e2.scheduler, Scheduler)
s.listen(8042)
e3 = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e3._start()
assert isinstance(e3.center, rpc)
assert isinstance(e3.scheduler, rpc)
s.stop()
yield e1._shutdown()
yield e2._shutdown()
yield e3._shutdown()
def f():
yield a._start()
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
e2 = Executor(e1.scheduler, start=False, loop=loop)
yield e2._start()
x = e1.submit(inc, 1)
y = e2.submit(inc, 2)
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
yield e1._restart()
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e1._shutdown(fast=True)
yield e2._shutdown(fast=True)
c.stop()
def f():
yield a._start()
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
e2 = Executor(scheduler=e1.scheduler, start=False, loop=loop)
yield e2._start()
x = e1.submit(inc, 1)
y = e2.submit(inc, 2)
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
yield e1._restart()
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e1._shutdown(fast=True)
yield e2._shutdown(fast=True)
c.stop()
def test__scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in s.who_has['y'] or
b.address in s.who_has['y'])
assert s.who_has['y']
assert s.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert s.who_has[x.key]
assert (a.address in s.who_has[x.key] or
b.address in s.who_has[x.key])
assert s.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert e.scheduler.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip},
{a.ip, b.ip},
{b.ip}])
yield _wait(L)
assert e.scheduler.restrictions[L[0].key] == {a.ip}
assert e.scheduler.restrictions[L[1].key] == {a.ip, b.ip}
assert e.scheduler.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def f():
yield a._start()
yield b._start()
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
yield e._restart()
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in e.scheduler.who_has['y'] or
b.address in e.scheduler.who_has['y'])
assert c.who_has['y']
assert e.scheduler.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert c.who_has[x.key]
assert (a.address in e.scheduler.who_has[x.key] or
b.address in e.scheduler.who_has[x.key])
assert e.scheduler.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def test_restrictions_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert s.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip},
{a.ip, b.ip},
{b.ip}])
yield _wait(L)
assert s.restrictions[L[0].key] == {a.ip}
assert s.restrictions[L[1].key] == {a.ip, b.ip}
assert s.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def test_future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
assert str(x.key) in repr(x)
assert str(x.status) in repr(x)
yield e._shutdown()
def test_future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
assert str(x.key) in repr(x)
assert str(x.status) in repr(x)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(randint, 0, 1000, pure=True)
xx = yield x._result()
f = Executor((c.ip, c.port), start=False, loop=loop)
yield f._start()
y = f.submit(randint, 0, 1000, pure=True)
yy = yield y._result()
assert xx == yy
yield e._shutdown()
yield f._shutdown()
def test_upload_file_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
with tmp_text('myfile.py', 'syntax-error!') as fn:
with pytest.raises(SyntaxError):
yield e._upload_file(fn)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
[x] = yield e._get({'x': (inc, 1)}, ['x'])
import gc; gc.collect()
assert e.refcount['x'] == 0
yield e._shutdown()
def test_directed_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def test_get_releases_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._get({'x': (inc, 1)}, ['x'])
import gc; gc.collect()
assert e.refcount['x'] == 0
yield e._shutdown()
def test_upload_file_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
with tmp_text('myfile.py', 'syntax-error!') as fn:
with pytest.raises(SyntaxError):
yield e._upload_file(fn)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
with tmp_text('myfile.py', 'syntax-error!') as fn:
with pytest.raises(SyntaxError):
yield e._upload_file(fn)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
with pytest.raises(ZeroDivisionError):
y = yield e._get(dsk, 'y')
yield e._shutdown()
def test_get_releases_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._get({'x': (inc, 1)}, ['x'])
import gc; gc.collect()
assert e.refcount['x'] == 0
yield e._shutdown()
def test_two_consecutive_executors_share_results(s, a, b):
from random import randint
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(randint, 0, 1000, pure=True)
xx = yield x._result()
f = Executor((s.ip, s.port), start=False)
yield f._start()
y = f.submit(randint, 0, 1000, pure=True)
yy = yield y._result()
assert xx == yy
yield e._shutdown()
yield f._shutdown()
def test_directed_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def test_two_consecutive_executors_share_results(s, a, b):
from random import randint
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(randint, 0, 1000, pure=True)
xx = yield x._result()
f = Executor((s.ip, s.port), start=False)
yield f._start()
y = f.submit(randint, 0, 1000, pure=True)
yy = yield y._result()
assert xx == yy
yield e._shutdown()
yield f._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def test_get_with_error(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
with pytest.raises(ZeroDivisionError):
y = yield e._get(dsk, 'y')
yield e._shutdown()
def test_get_with_error(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
with pytest.raises(ZeroDivisionError):
y = yield e._get(dsk, 'y')
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
L = [e.submit(inc, i) for i in range(10)]
yield _wait(L)
assert a.data and b.data
yield e._shutdown()
def test_many_submits_spread_evenly(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = [e.submit(inc, i) for i in range(10)]
yield _wait(L)
assert a.data and b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False,
loop=loop)
yield e._start()
from time import sleep
x = e.submit(sleep, 3)
yield x._result()
yield e._shutdown()
def test_remote_submit_on_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(lambda x: x + 1, 1)
y = e.submit(lambda x: x + 1, x)
result = yield y._result()
assert result == 3
yield e._shutdown()
def test_many_submits_spread_evenly(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = [e.submit(inc, i) for i in range(10)]
yield _wait(L)
assert a.data and b.data
yield e._shutdown()
def test_remote_submit_on_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(lambda x: x + 1, 1)
y = e.submit(lambda x: x + 1, x)
result = yield y._result()
assert result == 3
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), delete_batch_time=0, start=False,
loop=loop)
yield e._start()
from time import sleep
x = e.submit(sleep, 3)
yield x._result()
yield e._shutdown()
def test_submit_then_get_with_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(slowinc, 1)
dsk = {'y': (inc, x)}
result = yield e._get(dsk, 'y')
assert result == 3
yield e._shutdown()
def test_submit_then_get_with_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(slowinc, 1)
dsk = {'y': (inc, x)}
result = yield e._get(dsk, 'y')
assert result == 3
yield e._shutdown()
def test_remote_scheduler(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
assert isinstance(e.scheduler_stream, IOStream)
assert s.streams
x = e.submit(inc, 1)
result = yield x._result()
yield e._shutdown()
def test_traceback(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(div, 1, 0)
tb = yield x._traceback()
if sys.version_info[0] >= 3:
assert any('x / y' in line for line in tb)
yield e._shutdown()
def test_traceback(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(div, 1, 0)
tb = yield x._traceback()
if sys.version_info[0] >= 3:
assert any('x / y' in line for line in tb)
yield e._shutdown()
def test_aliases(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1)
dsk = {'y': x}
result = yield e._get(dsk, 'y')
assert result == 2
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 1)
dsk = {'y': x}
result = yield e._get(dsk, 'y')
assert result == 2
yield e._shutdown()
def test_aliases(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1)
dsk = {'y': x}
result = yield e._get(dsk, 'y')
assert result == 2
yield e._shutdown()
def test_badly_serialized_input(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
o = BadlySerializedObject()
future = e.submit(inc, o)
futures = e.map(inc, range(10))
L = yield e._gather(futures)
assert list(L) == list(map(inc, range(10)))
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(div, 1, 0)
tb = yield x._traceback()
if sys.version_info[0] >= 3:
assert any('x / y' in line for line in tb)
yield e._shutdown()
def test_remote_scheduler(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
assert isinstance(e.scheduler_stream, IOStream)
assert s.streams
x = e.submit(inc, 1)
result = yield x._result()
yield e._shutdown()
def test_Future_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(div, 1, 0)
result = yield x._exception()
assert isinstance(result, ZeroDivisionError)
x = e.submit(div, 1, 1)
result = yield x._exception()
assert result is None
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(div, 1, 0)
result = yield x._exception()
assert isinstance(result, ZeroDivisionError)
x = e.submit(div, 1, 1)
result = yield x._exception()
assert result is None
yield e._shutdown()
def test_multiple_executors(s, a, b):
a = Executor((s.ip, s.port), start=False)
yield a._start()
b = Executor((s.ip, s.port), start=False)
yield b._start()
x = a.submit(inc, 1)
y = b.submit(inc, 2)
assert x.executor is a
assert y.executor is b
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
z = a.submit(add, x, y)
assert z.executor is a
zz = yield z._result()
assert zz == 5
yield a._shutdown()
yield b._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
dsk = {('x', 0): (inc, 1), 5: (inc, 2)}
x = yield e._get(dsk, ('x', 0))
y = yield e._get(dsk, 5)
assert x == 2
assert y == 3
yield e._shutdown()
def f(c, a, b):
a = Executor((c.ip, c.port), start=False, loop=loop)
yield a._start()
b = Executor(scheduler=a.scheduler, start=False, loop=loop)
yield b._start()
x = a.submit(inc, 1)
y = b.submit(inc, 2)
assert x.executor is a
assert y.executor is b
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
z = a.submit(add, x, y)
assert z.executor is a
zz = yield z._result()
assert zz == 5
yield a._shutdown()
yield b._shutdown()
def test_get_with_non_list_key(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {('x', 0): (inc, 1), 5: (inc, 2)}
x = yield e._get(dsk, ('x', 0))
y = yield e._get(dsk, 5)
assert x == 2
assert y == 3
yield e._shutdown()
def test_restrictions_get(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')}
restrictions = {'y': {a.ip}, 'z': {b.ip}}
result = yield e._get(dsk, ['y', 'z'], restrictions)
assert result == [2, 3]
assert 'y' in a.data
assert 'z' in b.data
yield e._shutdown()
def dont_test_bad_restrictions_raise_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
z = e.submit(inc, 2, workers={'bad-address'})
try:
yield z._result()
assert False
except ValueError as e:
assert 'bad-address' in str(e)
assert z.key in str(e)
yield e._shutdown()
def test_submit_naming(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
a = e.submit(inc, 1)
b = e.submit(inc, 1)
assert a.event is b.event
c = e.submit(inc, 1, pure=False)
assert c.key != a.key
yield e._shutdown()
def test_nbytes_determines_worker(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(identity, 1, workers=[a.address[0]])
y = e.submit(identity, tuple(range(100)), workers=[b.address[0]])
yield e._gather([x, y])
z = e.submit(lambda x, y: None, x, y)
yield z._result()
assert s.who_has[z.key] == {b.address}
yield e._shutdown()
def test_gc(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
result = yield x._result()
assert s.who_has[x.key]
x.__del__()
yield e._shutdown()
assert not s.who_has[x.key]
def test_nbytes(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._scatter([1])
assert s.nbytes == {x.key: sizeof(1)}
y = e.submit(inc, x)
yield y._result()
assert s.nbytes == {x.key: sizeof(1),
y.key: sizeof(2)}
yield e._shutdown()
def test_remote_scatter_gather(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x, y, z = yield e._scatter([1, 2, 3])
assert x.key in a.data or x.key in b.data
assert y.key in a.data or y.key in b.data
assert z.key in a.data or z.key in b.data
xx, yy, zz = yield e._gather([x, y, z])
assert (xx, yy, zz) == (1, 2, 3)
yield e._shutdown()
def test_get(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
result = yield e._get({'x': (inc, 1)}, 'x')
assert result == 2
result = yield e._get({'x': (inc, 1)}, ['x'])
assert result == [2]
result = yield e._get({}, [])
assert result == []
yield e._shutdown()
def test_allow_restrictions(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1, workers=a.ip)
yield x._result()
assert s.who_has[x.key] == {a.address}
assert not s.loose_restrictions
x = e.submit(inc, 2, workers=a.ip, allow_other_workers=True)
yield x._result()
assert s.who_has[x.key] == {a.address}
assert x.key in s.loose_restrictions
L = e.map(inc, range(3, 13), workers=a.ip, allow_other_workers=True)
yield _wait(L)
assert all(s.who_has[f.key] == {a.address} for f in L)
assert {f.key for f in L}.issubset(s.loose_restrictions)
"""
x = e.submit(inc, 14, workers='127.0.0.3')
with ignoring(gen.TimeoutError):
yield gen.with_timeout(timedelta(seconds=0.1), x._result())
assert False
assert not s.who_has[x.key]
assert x.key not in s.loose_restrictions
"""
x = e.submit(inc, 15, workers='127.0.0.3', allow_other_workers=True)
yield x._result()
assert s.who_has[x.key]
assert x.key in s.loose_restrictions
L = e.map(inc, range(15, 25), workers='127.0.0.3', allow_other_workers=True)
yield _wait(L)
assert all(s.who_has[f.key] for f in L)
assert {f.key for f in L}.issubset(s.loose_restrictions)
with pytest.raises(ValueError):
e.submit(inc, 1, allow_other_workers=True)
with pytest.raises(ValueError):
e.map(inc, [1], allow_other_workers=True)
with pytest.raises(TypeError):
e.submit(inc, 20, workers='127.0.0.1', allow_other_workers='Hello!')
with pytest.raises(TypeError):
e.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!')
def test_restrictions_submit(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1, workers={a.ip})
y = e.submit(inc, x, workers={b.ip})
yield _wait([x, y])
assert s.restrictions[x.key] == {a.ip}
assert x.key in a.data
assert s.restrictions[y.key] == {b.ip}
assert y.key in b.data
yield e._shutdown()
