def test_nanny(s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start(0)
with rpc(n.address) as nn:
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.worker_info
yield nn.instantiate()
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
yield nn.terminate()
assert not n.is_alive()
yield n._close()
def test_nanny(s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start(0)
with rpc(n.address) as nn:
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.workers[n.worker_address].services['nanny'] > 1024
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.workers
yield nn.kill()
assert not n.is_alive()
assert n.worker_address not in s.ncores
assert n.worker_address not in s.workers
yield nn.instantiate()
assert n.is_alive()
assert s.ncores[n.worker_address] == 2
assert s.workers[n.worker_address].services['nanny'] > 1024
yield nn.terminate()
assert not n.is_alive()
yield n._close()
def test_many_kills(s):
n = Nanny(s.address, ncores=2, loop=s.loop)
yield n._start(0)
assert n.is_alive()
yield [n.kill() for i in range(5)]
yield [n.kill() for i in range(5)]
yield n._close()
def test_many_kills(s):
n = Nanny(s.address, ncores=2, loop=s.loop)
yield n._start(0)
assert n.is_alive()
yield [n.kill() for i in range(5)]
yield [n.kill() for i in range(5)]
yield n._close()
def test_nanny_process_failure(c, s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_address = n.worker_address
ww = rpc(n.worker_address)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
pid = n.pid
assert pid is not None
with ignoring(CommClosedError):
yield c._run(os._exit, 0, workers=[n.worker_address])
start = time()
while n.pid == pid: # wait while process dies and comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not n.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
# assert n.worker_address != original_address # most likely
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
ww.close_rpc()
s.stop()
def test_nanny_process_failure(c, s):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
original_address = n.worker_address
ww = rpc(n.worker_address)
yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
pid = n.pid
assert pid is not None
with ignoring(CommClosedError):
yield c._run(os._exit, 0, workers=[n.worker_address])
start = time()
while n.pid == pid: # wait while process dies and comes back
yield gen.sleep(0.01)
assert time() - start < 5
start = time()
while not n.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 5
# assert n.worker_address != original_address # most likely
start = time()
while n.worker_address not in s.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 5
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
ww.close_rpc()
s.stop()
