def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data={'x': 1, 'y': 2})
with ignoring(StreamClosedError):
yield ww.compute(function=sys.exit, args=(0,), key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores:
yield gen.sleep(0.01)
assert time() - start < 2
yield n._close()
c.stop()
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data={'x': 1, 'y': 2})
with ignoring(StreamClosedError):
yield ww.compute(function=sys.exit, args=(0,), key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 2
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
c.stop()
async def test_nanny_process_failure(c, s):
n = await Nanny(s.address, nthreads=2)
first_dir = n.worker_dir
assert os.path.exists(first_dir)
ww = rpc(n.worker_address)
await ww.update_data(data=valmap(dumps, {"x": 1, "y": 2}))
pid = n.pid
assert pid is not None
with suppress(CommClosedError):
await c.run(os._exit, 0, workers=[n.worker_address])
while n.pid == pid: # wait while process dies and comes back
await asyncio.sleep(0.01)
await asyncio.sleep(1)
while not n.is_alive(): # wait while process comes back
await asyncio.sleep(0.01)
# assert n.worker_address != original_address # most likely
while n.worker_address not in s.nthreads or n.worker_dir is None:
await asyncio.sleep(0.01)
second_dir = n.worker_dir
await n.close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
await ww.close_rpc()
s.stop()
def f():
scheduler = rpc(nannies[0].scheduler.address)
if nanny:
yield gen.with_timeout(timedelta(seconds=2),
All([scheduler.unregister(address=n.worker_address, close=True)
for n in nannies if n.process and n.worker_address]),
io_loop=loop2)
def f():
scheduler = rpc(ip=nannies[0].scheduler.ip,
port=nannies[0].scheduler.port)
if not no_nanny:
yield gen.with_timeout(timedelta(seconds=2),
All([scheduler.unregister(address=n.worker_address, close=True)
for n in nannies if n.process and n.worker_port]), io_loop=loop2)
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
assert n.process.is_alive()
assert c.ncores[n.worker_address] == 2
assert c.nannies[n.worker_address] > 8000
yield nn.kill()
assert n.worker_address not in c.ncores
assert n.worker_address not in c.nannies
assert not n.process
yield nn.kill()
assert n.worker_address not in c.ncores
assert n.worker_address not in c.nannies
assert not n.process
yield nn.instantiate()
assert n.process.is_alive()
assert c.ncores[n.worker_address] == 2
assert c.nannies[n.worker_address] > 8000
yield nn.terminate()
assert not n.process
if n.process:
n.process.terminate()
yield n._close()
c.stop()
def test_nanny(s):
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start(0)
nn = rpc(ip=n.ip, port=n.port)
assert n.process.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.kill()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
assert not n.process
yield nn.kill()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
assert not n.process
yield nn.instantiate()
assert n.process.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.terminate()
assert not n.process
yield n._close()
def test_nanny(s):
n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
with rpc(n.address) as nn:
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.workers[n.worker_address].services["nanny"] > 1024
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.workers
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.workers
yield nn.instantiate()
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.workers[n.worker_address].services["nanny"] > 1024
yield nn.terminate()
assert not n.is_alive()
yield n.close()
def test_monitor_resources():
pytest.importorskip('psutil')
c = Center(ip='127.0.0.1')
c.listen(0)
n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield n._start()
nn = rpc(ip=n.ip, port=n.port)
assert n.process.is_alive()
d = n.resource_collect()
assert {'cpu_percent', 'memory_percent'}.issubset(d)
assert isinstance(d['timestamp'], datetime)
stream = yield connect(ip=n.ip, port=n.port)
yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})
for i in range(3):
msg = yield read(stream)
assert isinstance(msg, dict)
assert {'cpu_percent', 'memory_percent'}.issubset(msg)
stream.close()
yield n._close()
c.stop()
def test_nanny():
c = Center('127.0.0.1')
c.listen(0)
n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield n._start(0)
nn = rpc(ip=n.ip, port=n.port)
assert n.process.is_alive()
assert c.ncores[n.worker_address] == 2
assert c.worker_services[n.worker_address]['nanny'] > 1024
yield nn.kill()
assert n.worker_address not in c.ncores
assert n.worker_address not in c.worker_services
assert not n.process
yield nn.kill()
assert n.worker_address not in c.ncores
assert n.worker_address not in c.worker_services
assert not n.process
yield nn.instantiate()
assert n.process.is_alive()
assert c.ncores[n.worker_address] == 2
assert c.worker_services[n.worker_address]['nanny'] > 1024
yield nn.terminate()
assert not n.process
if n.process:
n.process.terminate()
yield n._close()
c.stop()
def f():
yield a._start()
yield b._start()
e = Executor((c.ip, c.port), start=False, loop=loop)
yield e._start()
assert e.scheduler.ncores == {a.worker_address: 2, b.worker_address: 2}
x = e.submit(inc, 1)
y = e.submit(inc, x)
yield y._result()
cc = rpc(ip=c.ip, port=c.port)
who_has = yield cc.who_has()
try:
assert e.scheduler.who_has == who_has
assert set(e.scheduler.who_has) == {x.key, y.key}
yield e._restart()
assert len(e.scheduler.stacks) == 2
assert len(e.scheduler.processing) == 2
who_has = yield cc.who_has()
assert not who_has
assert not e.scheduler.who_has
assert x.cancelled()
assert y.cancelled()
finally:
yield a._close()
yield b._close()
yield e._shutdown(fast=True)
c.stop()
def test_nanny(s):
n = yield Nanny(s.address, nthreads=2, loop=s.loop)
with rpc(n.address) as nn:
assert n.is_alive()
assert s.nthreads[n.worker_address] == 2
assert s.workers[n.worker_address].nanny == n.address
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.nthreads
assert n.worker_address not in s.workers
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.nthreads
assert n.worker_address not in s.workers
yield nn.instantiate()
assert n.is_alive()
assert s.nthreads[n.worker_address] == 2
assert s.workers[n.worker_address].nanny == n.address
yield nn.terminate()
assert not n.is_alive()
yield n.close()
def test_nanny(s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start(0)
with rpc(n.address) as nn:
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
yield nn.instantiate()
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.terminate()
assert not n.is_alive()
yield n._close()
async def test_nanny(s):
async with Nanny(s.address, nthreads=2, loop=s.loop) as n:
async with rpc(n.address) as nn:
assert n.is_alive()
[ws] = s.workers.values()
assert ws.nthreads == 2
assert ws.nanny == n.address
await nn.kill()
assert not n.is_alive()
start = time()
while n.worker_address in s.workers:
assert time() < start + 1
await asyncio.sleep(0.01)
await nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.workers
await nn.instantiate()
assert n.is_alive()
[ws] = s.workers.values()
assert ws.nthreads == 2
assert ws.nanny == n.address
await nn.terminate()
assert not n.is_alive()
def test_nanny(s):
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start(0)
with rpc(ip=n.ip, port=n.port) as nn:
assert isalive(n.process) # alive
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.kill()
assert not n.process
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
yield nn.kill()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
assert not n.process
yield nn.instantiate()
assert isalive(n.process)
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.terminate()
assert not n.process
yield n._close()
def test_nanny(s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start(0)
with rpc(n.address) as nn:
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.workers[n.worker_address].services['nanny'] > 1024
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.workers
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.workers
yield nn.instantiate()
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.workers[n.worker_address].services['nanny'] > 1024
yield nn.terminate()
assert not n.is_alive()
yield n._close()
def _submit(remote_client_address, filepath, connection_args=None):
rc = rpc(remote_client_address, connection_args=connection_args)
remote_file = os.path.basename(filepath)
with open(filepath, 'rb') as f:
bytes_read = f.read()
yield rc.upload_file(filename=remote_file, file_payload=bytes_read)
result = yield rc.execute(filename=remote_file)
raise gen.Return((result['stdout'], result['stderr']))
def _submit(remote_client_address, filepath):
rc = rpc(addr=remote_client_address)
remote_file = os.path.basename(filepath)
with open(filepath, 'rb') as f:
bytes_read = f.read()
yield rc.upload_file(filename=remote_file, file_payload=bytes_read)
result = yield rc.execute(filename=remote_file)
raise gen.Return((result['stdout'], result['stderr']))
async def test_run(s):
n = await Nanny(s.address, nthreads=2, loop=s.loop)
with rpc(n.address) as nn:
response = await nn.run(function=dumps(lambda: 1))
assert response["status"] == "OK"
assert response["result"] == 1
await n.close()
def test_run(s):
pytest.importorskip('psutil')
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start()
nn = rpc(n.address)
response = yield nn.run(function=dumps(lambda: 1))
assert response['status'] == 'OK'
assert loads(response['result']) == 1
def test_run(s):
pytest.importorskip('psutil')
n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
with rpc(n.address) as nn:
response = yield nn.run(function=dumps(lambda: 1))
assert response['status'] == 'OK'
assert response['result'] == 1
yield n._close()
def test():
remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
yield remote_client._start(0)
remote_process = rpc(remote_client.address)
upload = yield remote_process.upload_file(filename='script.py', file_payload='x=1')
assert upload == {'status': 'OK', 'nbytes': 3}
assert tmpdir.join('script.py').read() == "x=1"
yield remote_client._close()
def test_run(s):
pytest.importorskip("psutil")
n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
with rpc(n.address) as nn:
response = yield nn.run(function=dumps(lambda: 1))
assert response["status"] == "OK"
assert response["result"] == 1
yield n.close()
def test():
remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
yield remote_client._start(0)
remote_process = rpc(remote_client.address)
upload = yield remote_process.upload_file(filename='script.py', file_payload='x=1')
assert upload == {'status': 'OK', 'nbytes': 3}
assert tmpdir.join('script.py').read() == "x=1"
yield remote_client._close()
def test():
remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
yield remote_client._start(0)
rr = rpc(remote_client.address)
yield rr.upload_file(filename='script.py', file_payload='print("hello world!")')
message = yield rr.execute(filename='script.py')
assert message['stdout'] == b'hello world!' + os.linesep.encode()
assert message['returncode'] == 0
yield remote_client._close()
def test():
remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
yield remote_client._start(0)
rr = rpc(remote_client.address)
yield rr.upload_file(filename='script.py', file_payload='a+1')
message = yield rr.execute(filename='script.py')
assert b'\'a\' is not defined' in message['stderr']
assert message['returncode'] == 1
yield remote_client._close()
def test():
remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
yield remote_client._start(0)
rr = rpc(remote_client.address)
yield rr.upload_file(filename="script.py", file_payload="a+1")
message = yield rr.execute(filename="script.py")
assert b"'a' is not defined" in message["stderr"]
assert message["returncode"] == 1
yield remote_client._close()
def test():
remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
yield remote_client._start(0)
rr = rpc(remote_client.address)
yield rr.upload_file(filename='script.py', file_payload='print("hello world!")')
message = yield rr.execute(filename='script.py')
assert message['stdout'] == b'hello world!' + os.linesep.encode()
assert message['returncode'] == 0
yield remote_client._close()
def test():
remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
yield remote_client._start(0)
rr = rpc(remote_client.address)
yield rr.upload_file(filename="script.py", file_payload='print("hello world!")')
message = yield rr.execute(filename="script.py")
assert message["stdout"] == b"hello world!" + os.linesep.encode()
assert message["returncode"] == 0
yield remote_client._close()
def test():
remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
yield remote_client._start(0)
rr = rpc(remote_client.address)
yield rr.upload_file(filename='script.py', file_payload='a+1')
message = yield rr.execute(filename='script.py')
assert b'\'a\' is not defined' in message['stderr']
assert message['returncode'] == 1
yield remote_client._close()
def test_run(s):
pytest.importorskip('psutil')
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start()
with rpc(n.address) as nn:
response = yield nn.run(function=dumps(lambda: 1))
assert response['status'] == 'OK'
assert response['result'] == 1
yield n._close()
def test():
remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
yield remote_client._start(0)
remote_process = rpc(remote_client.address)
upload = yield remote_process.upload_file(
filename="script.py", file_payload="x=1"
)
assert upload == {"status": "OK", "nbytes": 3}
assert tmpdir.join("script.py").read() == "x=1"
yield remote_client._close()
def test_nanny_process_failure():
c = Center('127.0.0.1')
c.listen(0)
n = Nanny(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield n._start()
nn = rpc(ip=n.ip, port=n.port)
first_dir = n.worker_dir
assert os.path.exists(first_dir)
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
with ignoring(StreamClosedError):
yield ww.compute(function=dumps(sys.exit),
args=dumps((0,)),
key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 2
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
nn.close_streams()
c.stop()
def test_nanny_process_failure(s):
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start()
nn = rpc(ip=n.ip, port=n.port)
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_process = n.process
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
with ignoring(StreamClosedError):
yield ww.compute(function=dumps(sys.exit),
args=dumps((0,)),
key='z')
start = time()
while n.process is original_process: # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not n.process.poll() is None: # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
nn.close_streams()
s.stop()
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
assert n.process.is_alive()
d = n.resource_collect()
assert {'cpu_percent', 'memory_percent'}.issubset(d)
assert isinstance(d['timestamp'], datetime)
stream = yield connect(ip=n.ip, port=n.port)
yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})
for i in range(3):
msg = yield read(stream)
assert isinstance(msg, dict)
assert {'cpu_percent', 'memory_percent'}.issubset(msg)
stream.close()
yield n._close()
c.stop()
def test_nanny_process_failure(c, s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_address = n.worker_address
ww = rpc(n.worker_address)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
pid = n.pid
assert pid is not None
with ignoring(CommClosedError):
yield c._run(os._exit, 0, workers=[n.worker_address])
start = time()
while n.pid == pid: # wait while process dies and comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not n.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
# assert n.worker_address != original_address # most likely
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
ww.close_rpc()
s.stop()
def test_nanny_process_failure(c, s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_address = n.worker_address
ww = rpc(n.worker_address)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
pid = n.pid
assert pid is not None
with ignoring(CommClosedError):
yield c._run(os._exit, 0, workers=[n.worker_address])
start = time()
while n.pid == pid: # wait while process dies and comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not n.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
# assert n.worker_address != original_address # most likely
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
ww.close_rpc()
s.stop()
def test_monitor_resources(s):
pytest.importorskip('psutil')
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start()
nn = rpc(ip=n.ip, port=n.port)
assert n.process.poll() is None
d = n.resource_collect()
assert {'cpu_percent', 'memory_percent'}.issubset(d)
assert 'timestamp' in d
stream = yield connect(ip=n.ip, port=n.port)
yield write(stream, {'op': 'monitor_resources', 'interval': 0.01})
for i in range(3):
msg = yield read(stream)
assert isinstance(msg, dict)
assert {'cpu_percent', 'memory_percent'}.issubset(msg)
stream.close()
yield n._close()
s.stop()
def test_nanny_process_failure(c, s):
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_process = n.process
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
with ignoring(StreamClosedError):
yield c._run(sys.exit, 0, workers=[n.worker_address])
start = time()
while n.process is original_process: # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not isalive(n.process): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
ww.close_rpc()
s.stop()
async def test_nanny(s):
async with Nanny(s.address, nthreads=2, loop=s.loop) as n:
async with rpc(n.address) as nn:
assert n.is_alive()
assert s.nthreads[n.worker_address] == 2
assert s.workers[n.worker_address].nanny == n.address
await nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.nthreads
assert n.worker_address not in s.workers
await nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.nthreads
assert n.worker_address not in s.workers
await nn.instantiate()
assert n.is_alive()
assert s.nthreads[n.worker_address] == 2
assert s.workers[n.worker_address].nanny == n.address
await nn.terminate()
assert not n.is_alive()
def test_nanny_process_failure(c, s):
n = Nanny(s.ip, s.port, ncores=2, ip='127.0.0.1', loop=s.loop)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_process = n.process
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
with ignoring(StreamClosedError):
yield c._run(sys.exit, 0, workers=[n.worker_address])
start = time()
while n.process is original_process: # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not isalive(n.process): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
ww.close_rpc()
s.stop()