def test__scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in s.who_has['y'] or
b.address in s.who_has['y'])
assert s.who_has['y']
assert s.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert s.who_has[x.key]
assert (a.address in s.who_has[x.key] or
b.address in s.who_has[x.key])
assert s.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def f():
yield a._start()
yield b._start()
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
yield e._restart()
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
def test_future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
assert str(x.key) in repr(x)
assert str(x.status) in repr(x)
yield e._shutdown()
def test_future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
assert str(x.key) in repr(x)
assert str(x.status) in repr(x)
yield e._shutdown()
def test_directed_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(assert_list, [1, 2, 3], z=[4, 5, 6])
result = yield x._result()
assert result
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
[x] = yield e._get({'x': (inc, 1)}, ['x'])
import gc; gc.collect()
assert e.refcount['x'] == 0
yield e._shutdown()
def test_upload_file_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
with tmp_text('myfile.py', 'syntax-error!') as fn:
with pytest.raises(SyntaxError):
yield e._upload_file(fn)
yield e._shutdown()
def test_directed_scatter(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def test_get_with_error(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
with pytest.raises(ZeroDivisionError):
y = yield e._get(dsk, 'y')
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(assert_list, [1, 2, 3], z=[4, 5, 6])
result = yield x._result()
assert result
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
yield e._scatter([1, 2, 3], workers=[a.address])
assert len(a.data) == 3
assert not b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
with pytest.raises(ZeroDivisionError):
y = yield e._get(dsk, 'y')
yield e._shutdown()
def test_get_releases_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._get({'x': (inc, 1)}, ['x'])
import gc; gc.collect()
assert e.refcount['x'] == 0
yield e._shutdown()
def test_get_with_error(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
with pytest.raises(ZeroDivisionError):
y = yield e._get(dsk, 'y')
yield e._shutdown()
def test_get_releases_data(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
[x] = yield e._get({'x': (inc, 1)}, ['x'])
import gc; gc.collect()
assert e.refcount['x'] == 0
yield e._shutdown()
def test_upload_file_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
with tmp_text('myfile.py', 'syntax-error!') as fn:
with pytest.raises(SyntaxError):
yield e._upload_file(fn)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
with tmp_text('myfile.py', 'syntax-error!') as fn:
with pytest.raises(SyntaxError):
yield e._upload_file(fn)
yield e._shutdown()
def test_many_submits_spread_evenly(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = [e.submit(inc, i) for i in range(10)]
yield _wait(L)
assert a.data and b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
L = [e.submit(inc, i) for i in range(10)]
yield _wait(L)
assert a.data and b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), delete_batch_time=0, start=False,
loop=loop)
yield e._start()
from time import sleep
x = e.submit(sleep, 3)
yield x._result()
yield e._shutdown()
def test_bad_address():
try:
Executor('123.123.123.123:1234', timeout=0.1)
except (IOError, gen.TimeoutError) as e:
assert "connect" in str(e).lower()
try:
Executor('127.0.0.1:1234', timeout=0.1)
except (IOError, gen.TimeoutError) as e:
assert "connect" in str(e).lower()
def test_remote_submit_on_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(lambda x: x + 1, 1)
y = e.submit(lambda x: x + 1, x)
result = yield y._result()
assert result == 3
yield e._shutdown()
def test_submit_errors():
def f(a, b, c):
pass
e = Executor('127.0.0.1:8787', start=False)
with pytest.raises(TypeError):
e.submit(1, 2, 3)
with pytest.raises(TypeError):
e.map([1, 2, 3])
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
a = e.submit(inc, 1)
b = e.submit(inc, 1)
assert a.event is b.event
c = e.submit(inc, 1, pure=False)
assert c.key != a.key
def test_remote_submit_on_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(lambda x: x + 1, 1)
y = e.submit(lambda x: x + 1, x)
result = yield y._result()
assert result == 3
yield e._shutdown()
def test_many_submits_spread_evenly(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = [e.submit(inc, i) for i in range(10)]
yield _wait(L)
assert a.data and b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False,
loop=loop)
yield e._start()
from time import sleep
x = e.submit(sleep, 3)
yield x._result()
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
a = e.submit(inc, 1)
b = e.submit(inc, 1)
assert a.event is b.event
c = e.submit(inc, 1, pure=False)
assert c.key != a.key
def f(c, a, b):
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
assert isinstance(e1.center, rpc)
assert isinstance(e1.scheduler, Scheduler)
s = Scheduler((c.ip, c.port))
yield s.sync_center()
done = s.start()
e2 = Executor(s, start=False, loop=loop)
yield e2._start()
assert isinstance(e2.center, rpc)
assert isinstance(e2.scheduler, Scheduler)
s.listen(8042)
e3 = Executor(('127.0.0.1', s.port), start=False, loop=loop)
yield e3._start()
assert isinstance(e3.center, rpc)
assert isinstance(e3.scheduler, rpc)
s.stop()
yield e1._shutdown()
yield e2._shutdown()
yield e3._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
d = yield e._scatter({'y': 20})
assert isinstance(d['y'], Future)
assert a.data.get('y') == 20 or b.data.get('y') == 20
assert (a.address in e.scheduler.who_has['y'] or
b.address in e.scheduler.who_has['y'])
assert c.who_has['y']
assert e.scheduler.nbytes == {'y': sizeof(20)}
yy = yield e._gather([d['y']])
assert yy == [20]
[x] = yield e._scatter([10])
assert isinstance(x, Future)
assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
xx = yield e._gather([x])
assert c.who_has[x.key]
assert (a.address in e.scheduler.who_has[x.key] or
b.address in e.scheduler.who_has[x.key])
assert e.scheduler.nbytes == {'y': sizeof(20), x.key: sizeof(10)}
assert xx == [10]
z = e.submit(add, x, d['y']) # submit works on RemoteData
result = yield z._result()
assert result == 10 + 20
result = yield e._gather([z, x])
assert result == [30, 10]
yield e._shutdown()
def f():
yield a._start()
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
e2 = Executor(scheduler=e1.scheduler, start=False, loop=loop)
yield e2._start()
x = e1.submit(inc, 1)
y = e2.submit(inc, 2)
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
yield e1._restart()
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e1._shutdown(fast=True)
yield e2._shutdown(fast=True)
c.stop()
def test_remote_scheduler(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
assert isinstance(e.scheduler_stream, IOStream)
assert s.streams
x = e.submit(inc, 1)
result = yield x._result()
yield e._shutdown()
def test_tokenize_on_futures():
e = Executor((None, None), start=False)
x = e.submit(inc, 1)
y = e.submit(inc, 1)
tok = tokenize(x)
assert tokenize(x) == tokenize(x)
assert tokenize(x) == tokenize(y)
e.futures[x.key]['status'] = 'finished'
assert tok == tokenize(y)
def test_traceback(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(div, 1, 0)
tb = yield x._traceback()
if sys.version_info[0] >= 3:
assert any('x / y' in line for line in tb)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(slowinc, 1)
dsk = {'y': (inc, x)}
result = yield e._get(dsk, 'y')
assert result == 3
yield e._shutdown()
def test_traceback(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(div, 1, 0)
tb = yield x._traceback()
if sys.version_info[0] >= 3:
assert any('x / y' in line for line in tb)
yield e._shutdown()
def test_remote_scheduler(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
assert isinstance(e.scheduler_stream, IOStream)
assert s.streams
x = e.submit(inc, 1)
result = yield x._result()
yield e._shutdown()
def test_submit_then_get_with_Future(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(slowinc, 1)
dsk = {'y': (inc, x)}
result = yield e._get(dsk, 'y')
assert result == 3
yield e._shutdown()
def test_aliases(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1)
dsk = {'y': x}
result = yield e._get(dsk, 'y')
assert result == 2
yield e._shutdown()
def test_badly_serialized_input(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
o = BadlySerializedObject()
future = e.submit(inc, o)
futures = e.map(inc, range(10))
L = yield e._gather(futures)
assert list(L) == list(map(inc, range(10)))
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(slowinc, 1)
dsk = {'y': (inc, x)}
result = yield e._get(dsk, 'y')
assert result == 3
yield e._shutdown()
def test_tokenize_on_futures():
e = Executor((None, None), start=False)
x = e.submit(inc, 1)
y = e.submit(inc, 1)
tok = tokenize(x)
assert tokenize(x) == tokenize(x)
assert tokenize(x) == tokenize(y)
e.futures[x.key]['status'] = 'finished'
assert tok == tokenize(y)
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(inc, 1)
dsk = {'y': x}
result = yield e._get(dsk, 'y')
assert result == 2
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(div, 1, 0)
tb = yield x._traceback()
if sys.version_info[0] >= 3:
assert any('x / y' in line for line in tb)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
x = e.submit(div, 1, 0)
result = yield x._exception()
assert isinstance(result, ZeroDivisionError)
x = e.submit(div, 1, 1)
result = yield x._exception()
assert result is None
yield e._shutdown()
def dont_test_bad_restrictions_raise_exception(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
z = e.submit(inc, 2, workers={'bad-address'})
try:
yield z._result()
assert False
except ValueError as e:
assert 'bad-address' in str(e)
assert z.key in str(e)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
dsk = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')}
restrictions = {'y': {a.ip}, 'z': {b.ip}}
result = yield e._get(dsk, 'z', restrictions)
assert result == 3
assert 'y' in a.data
assert 'z' in b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(inc, 10)
result = yield x._result()
assert c.who_has[x.key]
x.__del__()
yield e._shutdown()
assert not c.who_has[x.key]
def test_restrictions_get(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')}
restrictions = {'y': {a.ip}, 'z': {b.ip}}
result = yield e._get(dsk, ['y', 'z'], restrictions)
assert result == [2, 3]
assert 'y' in a.data
assert 'z' in b.data
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
z = e.submit(inc, 2, workers={'bad-address'})
try:
yield z._result()
assert False
except ValueError as e:
assert 'bad-address' in str(e)
assert z.key in str(e)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
L1 = e.map(inc, range(5))
L2 = e.map(inc, range(5))
assert [x.key for x in L1] == [x.key for x in L2]
L3 = e.map(inc, [1, 1, 1, 1])
assert len({x.event for x in L3}) == 1
L4 = e.map(inc, [1, 1, 1, 1], pure=False)
assert len({x.event for x in L4}) == 4
def test_restrictions_map(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(inc, range(5), workers={a.ip})
yield _wait(L)
assert set(a.data) == {x.key for x in L}
assert not b.data
for x in L:
assert s.restrictions[x.key] == {a.ip}
L = e.map(inc, [10, 11, 12], workers=[{a.ip},
{a.ip, b.ip},
{b.ip}])
yield _wait(L)
assert s.restrictions[L[0].key] == {a.ip}
assert s.restrictions[L[1].key] == {a.ip, b.ip}
assert s.restrictions[L[2].key] == {b.ip}
with pytest.raises(ValueError):
e.map(inc, [10, 11, 12], workers=[{a.ip}])
yield e._shutdown()
def test_gc(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 10)
result = yield x._result()
assert s.who_has[x.key]
x.__del__()
yield e._shutdown()
assert not s.who_has[x.key]
def test_global_executors(loop):
assert not _global_executor[0]
with pytest.raises(ValueError):
default_executor()
with cluster() as (s, [a, b]):
with Executor(('127.0.0.1', s['port']), loop=loop) as e:
assert _global_executor == [e]
assert default_executor() is e
with Executor(('127.0.0.1', s['port']), loop=loop) as f:
assert _global_executor == [f]
assert default_executor() is f
assert default_executor(e) is e
assert default_executor(f) is f
assert not _global_executor[0]
def test_restrictions_submit(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1, workers={a.ip})
y = e.submit(inc, x, workers={b.ip})
yield _wait([x, y])
assert s.restrictions[x.key] == {a.ip}
assert x.key in a.data
assert s.restrictions[y.key] == {b.ip}
assert y.key in b.data
yield e._shutdown()
def test_tokenize_on_futures(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
x = e.submit(inc, 1)
y = e.submit(inc, 1)
tok = tokenize(x)
assert tokenize(x) == tokenize(x)
assert tokenize(x) == tokenize(y)
e.futures[x.key]['status'] = 'finished'
assert tok == tokenize(y)
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
a = e.submit(inc, 1)
b = e.submit(inc, 1)
c = e.submit(inc, 2)
done, not_done = yield _wait([a, b, c])
assert done == {a, b, c}
assert not_done == set()
assert a.status == b.status == 'finished'
yield e._shutdown()
def f(c, a, b):
e = Executor((c.ip, c.port), start=False)
IOLoop.current().spawn_callback(e._go)
x = e.submit(inc, 1, workers={a.ip})
y = e.submit(inc, x, workers={b.ip})
yield _wait([x, y])
assert e.restrictions[x.key] == {a.ip}
assert x.key in a.data
assert e.restrictions[y.key] == {b.ip}
assert y.key in b.data
yield e._shutdown()
def test_missing_worker(s, a, b):
bad = ('bad-host', 8788)
s.ncores[bad] = 4
s.who_has['b'] = {bad}
s.has_what[bad] = {'b'}
e = Executor((s.ip, s.port), start=False)
yield e._start()
dsk = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}
result = yield e._get(dsk, 'c')
assert result == 3
assert bad not in s.ncores
yield e._shutdown()