def cluster(nworkers=2):
_port[0] += 1
cport = _port[0]
center = Process(target=run_center, args=(cport,))
workers = []
for i in range(nworkers):
_port[0] += 1
port = _port[0]
proc = Process(target=run_worker, args=(port, cport), kwargs={'ncores': 1})
workers.append({'port': port, 'proc': proc})
center.start()
for worker in workers:
worker['proc'].start()
sock = connect_sync('127.0.0.1', cport)
while True:
write_sync(sock, {'op': 'ncores'})
ncores = read_sync(sock)
if len(ncores) == nworkers:
break
try:
yield {'proc': center, 'port': cport}, workers
finally:
for port in [cport] + [w['port'] for w in workers]:
with ignoring(socket.error):
sock = connect_sync('127.0.0.1', port)
write_sync(sock, dict(op='terminate', close=True))
response = read_sync(sock)
sock.close()
for proc in [center] + [w['proc'] for w in workers]:
with ignoring(Exception):
proc.terminate()
def cluster(nworkers=2):
_port[0] += 1
cport = _port[0]
center = Process(target=run_center, args=(cport,))
workers = []
for i in range(nworkers):
_port[0] += 1
port = _port[0]
proc = Process(target=run_worker, args=(port, cport), kwargs={"ncores": 1})
workers.append({"port": port, "proc": proc})
center.start()
for worker in workers:
worker["proc"].start()
sock = connect_sync("127.0.0.1", cport)
while True:
write_sync(sock, {"op": "ncores"})
ncores = read_sync(sock)
if len(ncores) == nworkers:
break
try:
yield {"proc": center, "port": cport}, workers
finally:
for port in [cport] + [w["port"] for w in workers]:
with ignoring(socket.error):
sock = connect_sync("127.0.0.1", port)
write_sync(sock, dict(op="terminate", close=True))
response = read_sync(sock)
sock.close()
for proc in [center] + [w["proc"] for w in workers]:
with ignoring(Exception):
proc.terminate()
def test_cluster():
with Cluster('127.0.0.1', ['127.0.0.1', '127.0.0.1']) as c:
stream = connect_sync('127.0.0.1', 8787)
result = []
while len(result) != 2:
write_sync(stream, {'op': 'ncores'})
result = read_sync(stream)
c.add_worker('127.0.0.1')
while len(result) != 3:
write_sync(stream, {'op': 'ncores'})
result = read_sync(stream)
def cluster(nworkers=2, nanny=False):
if nanny:
_run_worker = run_nanny
else:
_run_worker = run_worker
_port[0] += 1
cport = _port[0]
center = Process(target=run_center, args=(cport,))
workers = []
for i in range(nworkers):
_port[0] += 1
port = _port[0]
proc = Process(target=_run_worker, args=(port, cport),
kwargs={'ncores': 1, 'local_dir': '_test_worker-%d' % port})
workers.append({'port': port, 'proc': proc})
center.start()
for worker in workers:
worker['proc'].start()
sock = connect_sync('127.0.0.1', cport)
start = time()
try:
while True:
write_sync(sock, {'op': 'ncores'})
ncores = read_sync(sock)
if len(ncores) == nworkers:
break
if time() - start > 5:
raise Exception("Timeout on cluster creation")
yield {'proc': center, 'port': cport}, workers
finally:
logger.debug("Closing out test cluster")
for port in [cport] + [w['port'] for w in workers]:
with ignoring(socket.error):
sock = connect_sync('127.0.0.1', port)
write_sync(sock, dict(op='terminate', close=True))
response = read_sync(sock)
sock.close()
for proc in [center] + [w['proc'] for w in workers]:
with ignoring(Exception):
proc.terminate()
for fn in glob('_test_worker-*'):
shutil.rmtree(fn)
def cluster(nworkers=2, nanny=False):
if nanny:
_run_worker = run_nanny
else:
_run_worker = run_worker
_port[0] += 1
cport = _port[0]
center = Process(target=run_center, args=(cport,))
workers = []
for i in range(nworkers):
_port[0] += 1
port = _port[0]
proc = Process(target=_run_worker, args=(port, cport), kwargs={"ncores": 1})
workers.append({"port": port, "proc": proc})
center.start()
for worker in workers:
worker["proc"].start()
sock = connect_sync("127.0.0.1", cport)
start = time()
try:
while True:
write_sync(sock, {"op": "ncores"})
ncores = read_sync(sock)
if len(ncores) == nworkers:
break
if time() - start > 5:
raise Exception("Timeout on cluster creation")
yield {"proc": center, "port": cport}, workers
finally:
logger.debug("Closing out test cluster")
for port in [cport] + [w["port"] for w in workers]:
with ignoring(socket.error):
sock = connect_sync("127.0.0.1", port)
write_sync(sock, dict(op="terminate", close=True))
response = read_sync(sock)
sock.close()
for proc in [center] + [w["proc"] for w in workers]:
with ignoring(Exception):
proc.terminate()
def cluster(nworkers=2):
_port[0] += 1
cport = _port[0]
center = Process(target=run_center, args=(cport, ))
workers = []
for i in range(nworkers):
_port[0] += 1
port = _port[0]
proc = Process(target=run_worker,
args=(port, cport),
kwargs={'ncores': 1})
workers.append({'port': port, 'proc': proc})
center.start()
for worker in workers:
worker['proc'].start()
sock = connect_sync('127.0.0.1', cport)
while True:
write_sync(sock, {'op': 'ncores'})
ncores = read_sync(sock)
if len(ncores) == nworkers:
break
try:
yield {'proc': center, 'port': cport}, workers
finally:
for port in [cport] + [w['port'] for w in workers]:
with ignoring(socket.error):
sock = connect_sync('127.0.0.1', port)
write_sync(sock, dict(op='terminate', close=True))
response = read_sync(sock)
sock.close()
for proc in [center] + [w['proc'] for w in workers]:
with ignoring(Exception):
proc.terminate()
def test_sync(loop):
def f():
from distributed.core import Server
from tornado.ioloop import IOLoop
server = Server({'ping': pingpong})
server.listen(8887)
IOLoop.current().start()
IOLoop.current().stop()
p = Process(target=f)
p.start()
try:
sock = connect_sync('127.0.0.1', 8887)
write_sync(sock, {'op': 'ping', 'close': True})
response = read_sync(sock)
assert response == b'pong'
finally:
p.terminate()
def test_sync():
def f():
from distributed.core import Server
from tornado.ioloop import IOLoop
server = Server({'ping': pingpong})
server.listen(8887)
IOLoop.current().start()
IOLoop.current().stop()
p = Process(target=f)
p.start()
try:
sock = connect_sync('127.0.0.1', 8887)
write_sync(sock, {'op': 'ping', 'close': True})
response = read_sync(sock)
assert response == b'pong'
finally:
p.terminate()
def test_sync():
def f():
from distributed.core import Server
from tornado.ioloop import IOLoop
server = Server({"ping": pingpong})
server.listen(8887)
IOLoop.current().start()
IOLoop.current().stop()
p = Process(target=f)
p.start()
try:
sock = connect_sync("127.0.0.1", 8887)
write_sync(sock, {"op": "ping", "close": True})
response = read_sync(sock)
assert response == b"pong"
finally:
p.terminate()