def test_no_workers_to_memory(c, s):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while not s.workers:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def get_repl_worker():
loop = IOLoop.current()
t = Thread(target=loop.start, daemon=True)
t.start()
w = Worker('tcp://127.0.0.1:8786', loop=loop)
w.start() # choose randomly assigned port
return w
def test_no_worker_to_memory_restrictions(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'})
while not s.tasks:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1, name='alice')
w.put_key_in_memory(y.key, 3)
yield w._start()
while len(s.workers) < 3:
yield gen.sleep(0.01)
yield gen.sleep(0.3)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def g():
c = Center('127.0.0.1')
c.listen(0)
a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield a._start()
b = Worker(c.ip, c.port, ncores=1, ip=b_ip)
yield b._start()
start = time()
try:
while len(c.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(c, a, b)
except Exception as e:
logger.exception(e)
raise
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
c.stop()
def test_worker_arrives_with_processing_data(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z])
while not any(w.processing for w in s.workers.values()):
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1)
w.put_key_in_memory(y.key, 3)
yield w._start()
start = time()
while len(s.workers) < 3:
yield gen.sleep(0.01)
assert s.get_task_status(keys={x.key, y.key, z.key}) == {
x.key: 'released',
y.key: 'memory',
z.key: 'processing',
}
yield w._close()
def g():
s = Scheduler(ip='127.0.0.1')
done = s.start()
s.listen(0)
a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1')
yield a._start()
b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip)
yield b._start()
start = time()
try:
while len(s.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(s, a, b)
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
yield s.close()
def test_services(s, a, b):
c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1',
services={'http': HTTPWorker})
yield c._start()
assert isinstance(c.services['http'], HTTPServer)
assert c.service_ports['http'] == c.services['http'].port
assert s.worker_services[c.address]['http'] == c.service_ports['http']
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop
import logging
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
worker.start()
IOLoop.current().start()
IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
def test_scatter_no_workers(s):
with pytest.raises(gen.TimeoutError):
yield gen.with_timeout(timedelta(seconds=0.1),
s.scatter(data={'x': dumps(1)}, client='alice'))
w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
yield [s.scatter(data={'x': dumps(1)}, client='alice'),
w._start()]
assert w.data['x'] == 1
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
worker.start()
loop.start()
def test_services_port(s, a, b):
c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1',
services={('http', 9898): HTTPWorker})
yield c._start()
assert isinstance(c.services['http'], HTTPServer)
assert (c.service_ports['http']
== c.services['http'].port
== s.worker_info[c.address]['services']['http']
== 9898)
c.services['http'].stop()
yield c._close()
def run_worker(q, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
loop.start()
def test_worker_name():
s = Scheduler()
s.start(0)
w = Worker(s.ip, s.port, name='alice')
yield w._start()
assert s.worker_info[w.address]['name'] == 'alice'
assert s.aliases['alice'] == w.address
with pytest.raises(ValueError):
w = Worker(s.ip, s.port, name='alice')
yield w._start()
yield s.close()
yield w._close()
def test_scatter_no_workers(c, s):
with pytest.raises(gen.TimeoutError):
yield s.scatter(data={'x': 1}, client='alice', timeout=0.1)
start = time()
with pytest.raises(gen.TimeoutError):
yield c.scatter(123, timeout=0.1)
assert time() < start + 1.5
w = Worker(s.ip, s.port, ncores=3)
yield [c.scatter(data={'y': 2}, timeout=5),
w._start()]
assert w.data['y'] == 2
yield w._close()
def run_worker(q, scheduler_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1',
loop=loop, validate=True, **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def test_file_descriptors_dont_leak(s):
psutil = pytest.importorskip('psutil')
proc = psutil.Process()
before = proc.num_fds()
w = Worker(s.ip, s.port)
yield w._start(0)
yield w._close()
during = proc.num_fds()
start = time()
while proc.num_fds() > before:
yield gen.sleep(0.01)
assert time() < start + 5
def test_add_worker(s, a, b):
w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
w.data[('x', 5)] = 6
w.data['y'] = 1
yield w._start(8020)
dsk = {('x', i): (inc, i) for i in range(10)}
s.update_graph(dsk=dsk, keys=list(dsk))
s.add_worker(address=w.address, keys=list(w.data),
ncores=w.ncores, nanny_port=w.nanny_port)
for k in w.data:
assert w.address in s.who_has[k]
s.validate(allow_overlap=True)
def test_add_worker(s, a, b):
w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
w.data[('x', 5)] = 6
w.data['y'] = 1
yield w._start(0)
dsk = {('x', i): (inc, i) for i in range(10)}
s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
dependencies={k: set() for k in dsk})
s.add_worker(address=w.address, keys=list(w.data),
ncores=w.ncores, services=s.services)
for k in w.data:
assert w.address in s.who_has[k]
s.validate(allow_overlap=True)
def test_add_worker(s, a, b):
w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
w.data['x-5'] = 6
w.data['y'] = 1
yield w._start(0)
dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)}
s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
dependencies={k: set() for k in dsk})
s.add_worker(address=w.address, keys=list(w.data),
ncores=w.ncores, services=s.services, coerce_address=False)
s.validate()
assert w.ip in s.host_info
assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port]))
def run_worker(q, scheduler_q, **kwargs):
from distributed import Worker
reset_logger_locks()
with log_errors():
with pristine_loop() as loop:
scheduler_addr = scheduler_q.get()
worker = Worker(scheduler_addr, validate=True, **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.address)
try:
@gen.coroutine
def wait_until_closed():
yield worker._closed.wait()
loop.run_sync(wait_until_closed)
finally:
loop.close(all_fds=True)
def test_add_worker(s, a, b):
w = Worker(s.ip, s.port, ncores=3)
w.data['x-5'] = 6
w.data['y'] = 1
yield w._start(0)
dsk = {('x-%d' % i): (inc, i) for i in range(10)}
s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
dependencies={k: set() for k in dsk})
s.add_worker(address=w.address, keys=list(w.data),
ncores=w.ncores, services=s.services)
s.validate_state()
assert w.ip in s.host_info
assert s.host_info[w.ip]['addresses'] == {a.address, b.address, w.address}
yield w._close()
def test_ready_add_worker(s, a, b):
s.add_client(client='client')
s.update_graph(tasks={'x-%d' % i: dumps_task((inc, i)) for i in range(20)},
keys=['x-%d' % i for i in range(20)],
client='client',
dependencies={'x-%d' % i: [] for i in range(20)})
assert all(len(s.processing[w]) == s.ncores[w]
for w in s.ncores)
assert len(s.ready) + sum(map(len, s.processing.values())) == 20
w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
w.listen(0)
s.add_worker(address=w.address, ncores=w.ncores, coerce_address=False)
assert w.address in s.ncores
assert all(len(s.processing[w]) == s.ncores[w]
for w in s.ncores)
assert len(s.ready) + sum(map(len, s.processing.values())) == 20
def test_resource_submit(c, s, a, b):
x = c.submit(inc, 1, resources={'A': 3})
y = c.submit(inc, 2, resources={'B': 1})
z = c.submit(inc, 3, resources={'C': 2})
yield _wait(x)
assert x.key in a.data
yield _wait(y)
assert y.key in b.data
assert z.key in s.unrunnable
d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10})
yield d._start()
yield _wait(z)
assert z.key in d.data
yield d._close()
def test_resource_submit(c, s, a, b):
x = c.submit(inc, 1, resources={'A': 3})
y = c.submit(inc, 2, resources={'B': 1})
z = c.submit(inc, 3, resources={'C': 2})
yield wait(x)
assert x.key in a.data
yield wait(y)
assert y.key in b.data
assert s.get_task_status(keys=[z.key]) == {z.key: 'no-worker'}
d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10})
yield d._start()
yield wait(z)
assert z.key in d.data
yield d._close()
def test_new_worker_steals(c, s, a):
yield _wait(c.submit(slowinc, 1, delay=0.01)._result())
futures = c.map(slowinc, range(100), delay=0.05)
total = c.submit(sum, futures)
while len(a.task_state) < 10:
yield gen.sleep(0.01)
b = Worker(s.ip, s.port, loop=s.loop, ncores=1)
yield b._start()
result = yield total._result()
assert result == sum(map(inc, range(100)))
for w in [a, b]:
assert all(isinstance(v, int) for v in w.data.values())
assert b.data
yield b._close()
def test_steal_resource_restrictions(c, s, a):
future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
yield future._result()
futures = c.map(slowinc, range(100), delay=0.2, resources={'A': 1})
while len(a.task_state) < 101:
yield gen.sleep(0.01)
assert len(a.task_state) == 101
b = Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={'A': 4})
yield b._start()
start = time()
while not b.task_state or len(a.task_state) == 101:
yield gen.sleep(0.01)
assert time() < start + 3
assert len(b.task_state) > 0
assert len(a.task_state) < 101
yield b._close()
def test_coerce_address():
with dask.config.set({'distributed.comm.timeouts.connect': '100ms'}):
s = Scheduler(validate=True)
s.start(0)
print("scheduler:", s.address, s.listen_address)
a = Worker(s.ip, s.port, name='alice')
b = Worker(s.ip, s.port, name=123)
c = Worker('127.0.0.1', s.port, name='charlie')
yield [a._start(), b._start(), c._start()]
assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000'
assert s.coerce_address('tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000'
assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000')
assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000')
assert s.coerce_address(a.address) == a.address
# Aliases
assert s.coerce_address('alice') == a.address
assert s.coerce_address(123) == b.address
assert s.coerce_address('charlie') == c.address
assert s.coerce_hostname('127.0.0.1') == '127.0.0.1'
assert s.coerce_hostname('alice') == a.ip
assert s.coerce_hostname(123) == b.ip
assert s.coerce_hostname('charlie') == c.ip
assert s.coerce_hostname('jimmy') == 'jimmy'
assert s.coerce_address('zzzt:8000', resolve=False) == 'tcp://zzzt:8000'
yield s.close()
yield [w._close() for w in [a, b, c]]
def g():
c = Center('127.0.0.1', 8017)
c.listen(c.port)
a = Worker('127.0.0.2', 8018, c.ip, c.port, ncores=2)
yield a._start()
b = Worker('127.0.0.3', 8019, c.ip, c.port, ncores=1)
yield b._start()
start = time()
try:
while len(c.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(c, a, b)
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring():
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
c.stop()
def test_errors_dont_block():
c = Center('127.0.0.1')
c.listen(0)
w = Worker(c.ip, c.port, ncores=1, ip='127.0.0.1')
e = Executor((c.ip, c.port), start=False, loop=IOLoop.current())
yield w._start()
yield e._start()
L = [e.submit(inc, 1),
e.submit(throws, 1),
e.submit(inc, 2),
e.submit(throws, 2)]
start = time()
while not (L[0].status == L[2].status == 'finished'):
assert time() < start + 5
yield gen.sleep(0.01)
result = yield e._gather([L[0], L[2]])
assert result == [2, 3]
yield w._close()
c.stop()
def test_add_worker(s, a, b):
w = Worker(s.ip, s.port, ncores=3)
w.data['x-5'] = 6
w.data['y'] = 1
yield w._start(0)
dsk = {('x-%d' % i): (inc, i) for i in range(10)}
s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
dependencies={k: set() for k in dsk})
s.add_worker(address=w.address, keys=list(w.data),
ncores=w.ncores, services=s.services)
s.validate_state()
assert w.ip in s.host_info
assert s.host_info[w.ip]['addresses'] == {a.address, b.address, w.address}
yield w._close()
def start_cluster(ncores,
scheduler_addr,
loop,
security=None,
Worker=Worker,
scheduler_kwargs={},
worker_kwargs={}):
s = Scheduler(loop=loop,
validate=True,
security=security,
**scheduler_kwargs)
done = s.start(scheduler_addr)
workers = [
Worker(s.address,
ncores=ncore[1],
name=i,
security=security,
loop=loop,
validate=True,
**(merge(worker_kwargs, ncore[2])
if len(ncore) > 2 else worker_kwargs))
for i, ncore in enumerate(ncores)
]
for w in workers:
w.rpc = workers[0].rpc
yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]
start = time()
while (len(s.workers) < len(ncores)
or any(comm.comm is None for comm in s.worker_comms.values())):
yield gen.sleep(0.01)
if time() - start > 5:
yield [w._close(timeout=1) for w in workers]
yield s.close(fast=True)
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
def test_add_worker(s, a, b):
w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1')
w.data['x-5'] = 6
w.data['y'] = 1
yield w._start(0)
dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)}
s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
dependencies={k: set() for k in dsk})
s.add_worker(address=w.address, keys=list(w.data),
ncores=w.ncores, services=s.services, coerce_address=False)
s.validate_state()
assert w.ip in s.host_info
assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port]))
yield w._close()
async def test_steal_twice(c, s, a, b):
x = c.submit(inc, 1, workers=a.address)
await wait(x)
futures = [c.submit(slowadd, x, i, delay=0.2) for i in range(100)]
while len(s.tasks) < 100: # tasks are all allocated
await asyncio.sleep(0.01)
# Army of new workers arrives to help
workers = await asyncio.gather(
*[Worker(s.address, loop=s.loop) for _ in range(20)])
await wait(futures)
has_what = dict(s.has_what) # take snapshot
empty_workers = [w for w, keys in has_what.items() if not len(keys)]
if len(empty_workers) > 2:
pytest.fail("Too many workers without keys (%d out of %d)" %
(len(empty_workers), len(has_what)))
assert max(map(len, has_what.values())) < 30
await c._close()
await asyncio.gather(*[w.close() for w in workers])
def test_file_descriptors_dont_leak(loop):
psutil = pytest.importorskip('psutil')
proc = psutil.Process()
before = proc.num_fds()
s = Scheduler()
s.start(0)
w = Worker(s.ip, s.port)
@gen.coroutine
def f():
yield w._start(0)
yield w._close()
loop.run_sync(f)
during = proc.num_fds()
s.stop()
s.close()
start = time()
while proc.num_fds() > before:
loop.run_sync(lambda: gen.sleep(0.01))
assert time() < start + 5
def become_dask_worker(address, nanny=False, **kwargs):
"""Task function for becoming a dask.distributed Worker
Parameters
----------
address: str
The URL of the dask Scheduler.
**kwargs:
Any additional keyword arguments will be passed to the Worker constructor.
"""
shell = get_ipython()
kernel = shell.kernel
if getattr(kernel, 'dask_worker', None) is not None:
kernel.log.info("Dask worker is already running.")
return
from distributed import Worker, Nanny
if nanny:
w = Nanny(address, **kwargs)
else:
w = Worker(address, **kwargs)
shell.user_ns['dask_worker'] = shell.user_ns[
'distributed_worker'] = kernel.distributed_worker = w
kernel.io_loop.add_callback(w.start)
def test_scatter_delete():
c = Center('127.0.0.1', 8017, loop=loop)
a = Worker('127.0.0.1', 8018, c.ip, c.port, loop=loop, ncores=1)
b = Worker('127.0.0.1', 8019, c.ip, c.port, loop=loop, ncores=1)
@asyncio.coroutine
def f():
while len(c.ncores) < 2:
yield from asyncio.sleep(0.01, loop=loop)
data = yield from scatter_to_center(c.ip, c.port, [1, 2, 3], loop=loop)
assert merge(a.data, b.data) == \
{d.key: i for d, i in zip(data, [1, 2, 3])}
assert set(c.who_has) == {d.key for d in data}
assert all(len(v) == 1 for v in c.who_has.values())
assert [d.get() for d in data] == [1, 2, 3]
yield from data[0]._delete()
assert merge(a.data, b.data) == \
{d.key: i for d, i in zip(data[1:], [2, 3])}
assert data[0].key not in c.who_has
data = yield from scatter_to_workers(c.ip, c.port,
[a.address, b.address], [4, 5, 6], loop=loop)
m = merge(a.data, b.data)
for d, v in zip(data, [4, 5, 6]):
assert m[d.key] == v
yield from a._close()
yield from b._close()
yield from c._close()
loop.run_until_complete(asyncio.gather(c.go(), a.go(), b.go(), f()))
def test_no_worker_to_memory_restrictions(c, s, a, b):
x = delayed(slowinc)(1, delay=0.4)
y = delayed(slowinc)(x, delay=0.4)
z = delayed(slowinc)(y, delay=0.4)
yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'})
while not s.task_state:
yield gen.sleep(0.01)
w = Worker(s.ip, s.port, ncores=1, name='alice')
w.put_key_in_memory(y.key, 3)
yield w._start()
while len(s.workers) < 3:
yield gen.sleep(0.01)
yield gen.sleep(0.3)
assert s.task_state[y.key] == 'memory'
assert s.task_state[x.key] == 'released'
assert s.task_state[z.key] == 'processing'
yield w._close()
def g():
c = Center('127.0.0.1')
c.listen(0)
a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1')
yield a._start()
b = Worker(c.ip, c.port, ncores=1, ip=b_ip)
yield b._start()
start = time()
try:
while len(c.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(c, a, b)
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
c.stop()
def test_scheduler_sees_memory_limits(s):
w = yield Worker(s.ip, s.port, ncores=3, memory_limit=12345)
assert s.workers[w.address].memory_limit == 12345
yield w._close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 22 17:15:29 2019
@author: cliffk
"""
from distributed import Worker
from tornado.ioloop import IOLoop
from threading import Thread
loop = IOLoop.current()
t = Thread(target=loop.start, daemon=True)
t.start()
w = Worker('tcp://127.0.0.1:8786', loop=loop)
w.start() # choose randomly assigned port
setup_log_signal_handling()
logger = logging.getLogger('worker')
logging.basicConfig(
level=logging.DEBUG,
format=
'[%(asctime)s] [%(process)s/%(threadName)s] [%(levelname)s] [%(name)s] %(message)s'
)
scheduler_address = os.getenv('DASK_SCHEDULER_ADDRESS', "192.168.65.3")
scheduler_port = os.getenv('DASK_SCHEDULER_PORT', 8786)
death_time_out = 30
scheduler = "%s:%s" % (scheduler_address, scheduler_port)
worker = Worker(scheduler, death_timeout=death_time_out)
pid = os.getpid()
def worker_watchdog():
"""
A routine that watches the state of the Worker
that is spawned and that kills it when possible.
This is used to resolve a teardown problem when
the connection between the Scheduler and the Worker
ends.
:return:
"""
delay = 5 # seconds
async def test_security_dict_input_no_security(cleanup):
async with Scheduler(security={}) as s:
async with Worker(s.address, security={}) as w:
async with Client(s.address, security={}, asynchronous=True) as c:
result = await c.submit(inc, 1)
assert result == 2
# encoding: utf-8
import tornado.web
import tornado.ioloop
import tornado.gen
import json
import requests
import traceback
from cesium_app.config import cfg
IP = '127.0.0.1'
PORT_SCHEDULER = 63500
if __name__ == "__main__":
loop = tornado.ioloop.IOLoop.current()
from distributed import Scheduler
s = Scheduler(loop=loop)
s.start(PORT_SCHEDULER)
print('Task scheduler listening on port {}'.format(PORT_SCHEDULER))
from distributed import Worker
w = Worker('127.0.0.1', PORT_SCHEDULER, loop=loop, ncores=1)
w.start(0)
print('Single worker activated')
print('Starting main loop...')
loop.start()
def test_add_remove_worker(s):
events = []
class MyPlugin(SchedulerPlugin):
def add_worker(self, worker, scheduler):
assert scheduler is s
events.append(('add_worker', worker))
def remove_worker(self, worker, scheduler):
assert scheduler is s
events.append(('remove_worker', worker))
plugin = MyPlugin()
s.add_plugin(plugin)
assert events == []
a = Worker(s.address)
b = Worker(s.address)
yield a._start()
yield b._start()
yield a._close()
yield b._close()
assert events == [('add_worker', a.address),
('add_worker', b.address),
('remove_worker', a.address),
('remove_worker', b.address),
]
events[:] = []
s.remove_plugin(plugin)
a = Worker(s.address)
yield a._start()
yield a._close()
assert events == []
def test_scheduler_sees_memory_limits(s):
w = yield Worker(s.address, nthreads=3, memory_limit=12345)
assert s.workers[w.address].memory_limit == 12345
yield w.close()
def _run(cls, worker_args, worker_kwargs, worker_start_args,
silence_logs, init_result_q, child_stop_q): # pragma: no cover
from distributed import Worker
try:
from dask.multiprocessing import initialize_worker_process
except ImportError: # old Dask version
pass
else:
initialize_worker_process()
if silence_logs:
logger.setLevel(silence_logs)
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(*worker_args, **worker_kwargs)
@gen.coroutine
def do_stop(timeout):
try:
yield worker._close(report=False, nanny=False)
finally:
loop.stop()
def watch_stop_q():
"""
Wait for an incoming stop message and then stop the
worker cleanly.
"""
while True:
try:
msg = child_stop_q.get(timeout=1000)
except Empty:
pass
else:
assert msg['op'] == 'stop'
loop.add_callback(do_stop, msg['timeout'])
break
t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
t.daemon = True
t.start()
@gen.coroutine
def run():
"""
Try to start worker and inform parent of outcome.
"""
try:
yield worker._start(*worker_start_args)
except Exception as e:
logger.exception(e)
init_result_q.put(e)
else:
assert worker.address
init_result_q.put({'address': worker.address,
'dir': worker.local_dir})
yield worker.wait_until_closed()
logger.info("Worker closed")
try:
loop.run_sync(run)
except TimeoutError:
# Loop was stopped before wait_until_closed() returned, ignore
pass
except KeyboardInterrupt:
pass
async def test_simple(cleanup):
async with Scheduler(protocol="ucx") as s:
async with Worker(s.address) as a:
async with Client(s.address, asynchronous=True) as c:
result = await c.submit(lambda x: x + 1, 10)
assert result == 11
async def test_deprecated_params(s, name):
with pytest.warns(FutureWarning, match=name):
async with Worker(s.address, **{name: 0.789}) as a:
assert getattr(a.memory_manager, name) == 0.789
def test_scheduler_sees_memory_limits(s):
w = Worker(s.ip, s.port, ncores=3, memory_limit=12345)
yield w._start(0)
assert s.worker_info[w.address]['memory_limit'] == 12345
yield w._close()
async def test_large_transfer(cleanup):
np = pytest.importorskip("numpy")
async with Scheduler(protocol="ws://") as s:
async with Worker(s.address, protocol="ws://") as w:
async with Client(s.address, asynchronous=True) as c:
future = await c.scatter(np.random.random(1000000))
SCHEDULER_PORT = 5678
SCHEDULER_HTTP_PORT = 9786
SCHEDULER_BOKEH_PORT = 12345
SCHEDULER_IP = '127.0.0.1'
HOME_PAGE = 'http://localhost:5050'
print('Hello')
loop_worker = IOLoop()
t = Thread(target=loop_worker.start)
t.start()
NUM_WORKERS = 2
nanny_process = [
Worker(center_ip=SCHEDULER_IP,
center_port=SCHEDULER_PORT,
loop=loop_worker,
ncores=2) for _ in range(NUM_WORKERS)
]
for nanny in nanny_process:
nanny.start()
t.join()
for nanny in nanny_process:
nanny.start()
loop_worker.stop()
from distributed import Worker
#from distributed import Nanny
from tornado.ioloop import IOLoop
from threading import Thread
w = Worker('tcp://dscheduler:8786')
#w = Nanny('tcp://dscheduler:8786')
w.start() # choose randomly assigned port
loop = IOLoop.current()
loop.start()
