def f():
yield a._start()
yield b._start()
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
yield e._restart()
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
def f():
yield a._start()
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
e2 = Executor(scheduler=e1.scheduler, start=False, loop=loop)
yield e2._start()
x = e1.submit(inc, 1)
y = e2.submit(inc, 2)
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
yield e1._restart()
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e1._shutdown(fast=True)
yield e2._shutdown(fast=True)
c.stop()
def f():
yield a._start()
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
e1 = Executor((c.ip, c.port), start=False, loop=loop)
yield e1._start()
e2 = Executor(e1.scheduler, start=False, loop=loop)
yield e2._start()
x = e1.submit(inc, 1)
y = e2.submit(inc, 2)
xx = yield x._result()
yy = yield y._result()
assert xx == 2
assert yy == 3
yield e1._restart()
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e1._shutdown(fast=True)
yield e2._shutdown(fast=True)
c.stop()
def test_restart():
from distributed import Nanny, rpc
c = Center('127.0.0.1')
c.listen(0)
a = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
b = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield [a._start(), b._start()]
e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
f = yield e._restart()
assert f is e
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
def test_restart():
from distributed import Nanny, rpc
c = Center('127.0.0.1')
c.listen(0)
a = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
b = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield [a._start(), b._start()]
e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
f = yield e._restart()
assert f is e
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
def test_failed_worker_without_warning(s, a, b):
e = Executor((s.ip, s.port), start=False)
yield e._start()
L = e.map(inc, range(10))
yield _wait(L)
ncores1 = s.ncores.copy()
a.process.terminate()
start = time()
while not a.process.is_alive():
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while s.ncores == ncores1:
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while len(s.ncores) < 2:
yield gen.sleep(0.01)
assert time() - start < 5
yield _wait(L)
L2 = e.map(inc, range(10, 20))
yield _wait(L2)
assert all(len(keys) > 0 for keys in s.has_what.values())
ncores2 = s.ncores.copy()
yield e._restart()
L = e.map(inc, range(10))
yield _wait(L)
assert all(len(keys) > 0 for keys in s.has_what.values())
assert not (set(ncores2) & set(s.ncores)) # no overlap
