def test_no_workers_to_memory(c, s): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while not s.workers: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def get_repl_worker(): loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() w = Worker('tcp://127.0.0.1:8786', loop=loop) w.start() # choose randomly assigned port return w
def test_no_worker_to_memory_restrictions(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'}) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1, name='alice') w.put_key_in_memory(y.key, 3) yield w._start() while len(s.workers) < 3: yield gen.sleep(0.01) yield gen.sleep(0.3) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def g(): c = Center('127.0.0.1') c.listen(0) a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker(c.ip, c.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) except Exception as e: logger.exception(e) raise finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) c.stop()
def test_worker_arrives_with_processing_data(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not any(w.processing for w in s.workers.values()): yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while len(s.workers) < 3: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def g(): s = Scheduler(ip='127.0.0.1') done = s.start() s.listen(0) a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(s.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(s, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) yield s.close()
def test_services(s, a, b): c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1', services={'http': HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert c.service_ports['http'] == c.services['http'].port assert s.worker_services[c.address]['http'] == c.service_ports['http']
def run_worker(port, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop import logging logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs) worker.start() IOLoop.current().start() IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
def test_scatter_no_workers(s): with pytest.raises(gen.TimeoutError): yield gen.with_timeout(timedelta(seconds=0.1), s.scatter(data={'x': dumps(1)}, client='alice')) w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') yield [s.scatter(data={'x': dumps(1)}, client='alice'), w._start()] assert w.data['x'] == 1
def run_worker(port, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs) worker.start() loop.start()
def test_services_port(s, a, b): c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1', services={('http', 9898): HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert (c.service_ports['http'] == c.services['http'].port == s.worker_info[c.address]['services']['http'] == 9898) c.services['http'].stop() yield c._close()
def run_worker(q, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) loop.start()
def test_worker_name(): s = Scheduler() s.start(0) w = Worker(s.ip, s.port, name='alice') yield w._start() assert s.worker_info[w.address]['name'] == 'alice' assert s.aliases['alice'] == w.address with pytest.raises(ValueError): w = Worker(s.ip, s.port, name='alice') yield w._start() yield s.close() yield w._close()
def test_scatter_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield s.scatter(data={'x': 1}, client='alice', timeout=0.1) start = time() with pytest.raises(gen.TimeoutError): yield c.scatter(123, timeout=0.1) assert time() < start + 1.5 w = Worker(s.ip, s.port, ncores=3) yield [c.scatter(data={'y': 2}, timeout=5), w._start()] assert w.data['y'] == 2 yield w._close()
def run_worker(q, scheduler_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1', loop=loop, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.close(all_fds=True)
def test_file_descriptors_dont_leak(s): psutil = pytest.importorskip('psutil') proc = psutil.Process() before = proc.num_fds() w = Worker(s.ip, s.port) yield w._start(0) yield w._close() during = proc.num_fds() start = time() while proc.num_fds() > before: yield gen.sleep(0.01) assert time() < start + 5
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data[('x', 5)] = 6 w.data['y'] = 1 yield w._start(8020) dsk = {('x', i): (inc, i) for i in range(10)} s.update_graph(dsk=dsk, keys=list(dsk)) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, nanny_port=w.nanny_port) for k in w.data: assert w.address in s.who_has[k] s.validate(allow_overlap=True)
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data[('x', 5)] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x', i): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services) for k in w.data: assert w.address in s.who_has[k] s.validate(allow_overlap=True)
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services, coerce_address=False) s.validate() assert w.ip in s.host_info assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port]))
def run_worker(q, scheduler_q, **kwargs): from distributed import Worker reset_logger_locks() with log_errors(): with pristine_loop() as loop: scheduler_addr = scheduler_q.get() worker = Worker(scheduler_addr, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.address) try: @gen.coroutine def wait_until_closed(): yield worker._closed.wait() loop.run_sync(wait_until_closed) finally: loop.close(all_fds=True)
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3) w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services) s.validate_state() assert w.ip in s.host_info assert s.host_info[w.ip]['addresses'] == {a.address, b.address, w.address} yield w._close()
def test_ready_add_worker(s, a, b): s.add_client(client='client') s.update_graph(tasks={'x-%d' % i: dumps_task((inc, i)) for i in range(20)}, keys=['x-%d' % i for i in range(20)], client='client', dependencies={'x-%d' % i: [] for i in range(20)}) assert all(len(s.processing[w]) == s.ncores[w] for w in s.ncores) assert len(s.ready) + sum(map(len, s.processing.values())) == 20 w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.listen(0) s.add_worker(address=w.address, ncores=w.ncores, coerce_address=False) assert w.address in s.ncores assert all(len(s.processing[w]) == s.ncores[w] for w in s.ncores) assert len(s.ready) + sum(map(len, s.processing.values())) == 20
def test_resource_submit(c, s, a, b): x = c.submit(inc, 1, resources={'A': 3}) y = c.submit(inc, 2, resources={'B': 1}) z = c.submit(inc, 3, resources={'C': 2}) yield _wait(x) assert x.key in a.data yield _wait(y) assert y.key in b.data assert z.key in s.unrunnable d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10}) yield d._start() yield _wait(z) assert z.key in d.data yield d._close()
def test_resource_submit(c, s, a, b): x = c.submit(inc, 1, resources={'A': 3}) y = c.submit(inc, 2, resources={'B': 1}) z = c.submit(inc, 3, resources={'C': 2}) yield wait(x) assert x.key in a.data yield wait(y) assert y.key in b.data assert s.get_task_status(keys=[z.key]) == {z.key: 'no-worker'} d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10}) yield d._start() yield wait(z) assert z.key in d.data yield d._close()
def test_new_worker_steals(c, s, a): yield _wait(c.submit(slowinc, 1, delay=0.01)._result()) futures = c.map(slowinc, range(100), delay=0.05) total = c.submit(sum, futures) while len(a.task_state) < 10: yield gen.sleep(0.01) b = Worker(s.ip, s.port, loop=s.loop, ncores=1) yield b._start() result = yield total._result() assert result == sum(map(inc, range(100))) for w in [a, b]: assert all(isinstance(v, int) for v in w.data.values()) assert b.data yield b._close()
def test_steal_resource_restrictions(c, s, a): future = c.submit(slowinc, 1, delay=0.10, workers=a.address) yield future._result() futures = c.map(slowinc, range(100), delay=0.2, resources={'A': 1}) while len(a.task_state) < 101: yield gen.sleep(0.01) assert len(a.task_state) == 101 b = Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={'A': 4}) yield b._start() start = time() while not b.task_state or len(a.task_state) == 101: yield gen.sleep(0.01) assert time() < start + 3 assert len(b.task_state) > 0 assert len(a.task_state) < 101 yield b._close()
def test_coerce_address(): with dask.config.set({'distributed.comm.timeouts.connect': '100ms'}): s = Scheduler(validate=True) s.start(0) print("scheduler:", s.address, s.listen_address) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker('127.0.0.1', s.port, name='charlie') yield [a._start(), b._start(), c._start()] assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(a.address) == a.address # Aliases assert s.coerce_address('alice') == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address assert s.coerce_hostname('127.0.0.1') == '127.0.0.1' assert s.coerce_hostname('alice') == a.ip assert s.coerce_hostname(123) == b.ip assert s.coerce_hostname('charlie') == c.ip assert s.coerce_hostname('jimmy') == 'jimmy' assert s.coerce_address('zzzt:8000', resolve=False) == 'tcp://zzzt:8000' yield s.close() yield [w._close() for w in [a, b, c]]
def g(): c = Center('127.0.0.1', 8017) c.listen(c.port) a = Worker('127.0.0.2', 8018, c.ip, c.port, ncores=2) yield a._start() b = Worker('127.0.0.3', 8019, c.ip, c.port, ncores=1) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) c.stop()
def test_errors_dont_block(): c = Center('127.0.0.1') c.listen(0) w = Worker(c.ip, c.port, ncores=1, ip='127.0.0.1') e = Executor((c.ip, c.port), start=False, loop=IOLoop.current()) yield w._start() yield e._start() L = [e.submit(inc, 1), e.submit(throws, 1), e.submit(inc, 2), e.submit(throws, 2)] start = time() while not (L[0].status == L[2].status == 'finished'): assert time() < start + 5 yield gen.sleep(0.01) result = yield e._gather([L[0], L[2]]) assert result == [2, 3] yield w._close() c.stop()
def start_cluster(ncores, scheduler_addr, loop, security=None, Worker=Worker, scheduler_kwargs={}, worker_kwargs={}): s = Scheduler(loop=loop, validate=True, security=security, **scheduler_kwargs) done = s.start(scheduler_addr) workers = [ Worker(s.address, ncores=ncore[1], name=i, security=security, loop=loop, validate=True, **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)) for i, ncore in enumerate(ncores) ] for w in workers: w.rpc = workers[0].rpc yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)] start = time() while (len(s.workers) < len(ncores) or any(comm.comm is None for comm in s.worker_comms.values())): yield gen.sleep(0.01) if time() - start > 5: yield [w._close(timeout=1) for w in workers] yield s.close(fast=True) raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services, coerce_address=False) s.validate_state() assert w.ip in s.host_info assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port])) yield w._close()
async def test_steal_twice(c, s, a, b): x = c.submit(inc, 1, workers=a.address) await wait(x) futures = [c.submit(slowadd, x, i, delay=0.2) for i in range(100)] while len(s.tasks) < 100: # tasks are all allocated await asyncio.sleep(0.01) # Army of new workers arrives to help workers = await asyncio.gather( *[Worker(s.address, loop=s.loop) for _ in range(20)]) await wait(futures) has_what = dict(s.has_what) # take snapshot empty_workers = [w for w, keys in has_what.items() if not len(keys)] if len(empty_workers) > 2: pytest.fail("Too many workers without keys (%d out of %d)" % (len(empty_workers), len(has_what))) assert max(map(len, has_what.values())) < 30 await c._close() await asyncio.gather(*[w.close() for w in workers])
def test_file_descriptors_dont_leak(loop): psutil = pytest.importorskip('psutil') proc = psutil.Process() before = proc.num_fds() s = Scheduler() s.start(0) w = Worker(s.ip, s.port) @gen.coroutine def f(): yield w._start(0) yield w._close() loop.run_sync(f) during = proc.num_fds() s.stop() s.close() start = time() while proc.num_fds() > before: loop.run_sync(lambda: gen.sleep(0.01)) assert time() < start + 5
def become_dask_worker(address, nanny=False, **kwargs): """Task function for becoming a dask.distributed Worker Parameters ---------- address: str The URL of the dask Scheduler. **kwargs: Any additional keyword arguments will be passed to the Worker constructor. """ shell = get_ipython() kernel = shell.kernel if getattr(kernel, 'dask_worker', None) is not None: kernel.log.info("Dask worker is already running.") return from distributed import Worker, Nanny if nanny: w = Nanny(address, **kwargs) else: w = Worker(address, **kwargs) shell.user_ns['dask_worker'] = shell.user_ns[ 'distributed_worker'] = kernel.distributed_worker = w kernel.io_loop.add_callback(w.start)
def test_scatter_delete(): c = Center('127.0.0.1', 8017, loop=loop) a = Worker('127.0.0.1', 8018, c.ip, c.port, loop=loop, ncores=1) b = Worker('127.0.0.1', 8019, c.ip, c.port, loop=loop, ncores=1) @asyncio.coroutine def f(): while len(c.ncores) < 2: yield from asyncio.sleep(0.01, loop=loop) data = yield from scatter_to_center(c.ip, c.port, [1, 2, 3], loop=loop) assert merge(a.data, b.data) == \ {d.key: i for d, i in zip(data, [1, 2, 3])} assert set(c.who_has) == {d.key for d in data} assert all(len(v) == 1 for v in c.who_has.values()) assert [d.get() for d in data] == [1, 2, 3] yield from data[0]._delete() assert merge(a.data, b.data) == \ {d.key: i for d, i in zip(data[1:], [2, 3])} assert data[0].key not in c.who_has data = yield from scatter_to_workers(c.ip, c.port, [a.address, b.address], [4, 5, 6], loop=loop) m = merge(a.data, b.data) for d, v in zip(data, [4, 5, 6]): assert m[d.key] == v yield from a._close() yield from b._close() yield from c._close() loop.run_until_complete(asyncio.gather(c.go(), a.go(), b.go(), f()))
def test_no_worker_to_memory_restrictions(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'}) while not s.task_state: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1, name='alice') w.put_key_in_memory(y.key, 3) yield w._start() while len(s.workers) < 3: yield gen.sleep(0.01) yield gen.sleep(0.3) assert s.task_state[y.key] == 'memory' assert s.task_state[x.key] == 'released' assert s.task_state[z.key] == 'processing' yield w._close()
def g(): c = Center('127.0.0.1') c.listen(0) a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker(c.ip, c.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) c.stop()
def test_scheduler_sees_memory_limits(s): w = yield Worker(s.ip, s.port, ncores=3, memory_limit=12345) assert s.workers[w.address].memory_limit == 12345 yield w._close()
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Fri Feb 22 17:15:29 2019 @author: cliffk """ from distributed import Worker from tornado.ioloop import IOLoop from threading import Thread loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() w = Worker('tcp://127.0.0.1:8786', loop=loop) w.start() # choose randomly assigned port
setup_log_signal_handling() logger = logging.getLogger('worker') logging.basicConfig( level=logging.DEBUG, format= '[%(asctime)s] [%(process)s/%(threadName)s] [%(levelname)s] [%(name)s] %(message)s' ) scheduler_address = os.getenv('DASK_SCHEDULER_ADDRESS', "192.168.65.3") scheduler_port = os.getenv('DASK_SCHEDULER_PORT', 8786) death_time_out = 30 scheduler = "%s:%s" % (scheduler_address, scheduler_port) worker = Worker(scheduler, death_timeout=death_time_out) pid = os.getpid() def worker_watchdog(): """ A routine that watches the state of the Worker that is spawned and that kills it when possible. This is used to resolve a teardown problem when the connection between the Scheduler and the Worker ends. :return: """ delay = 5 # seconds
async def test_security_dict_input_no_security(cleanup): async with Scheduler(security={}) as s: async with Worker(s.address, security={}) as w: async with Client(s.address, security={}, asynchronous=True) as c: result = await c.submit(inc, 1) assert result == 2
# encoding: utf-8 import tornado.web import tornado.ioloop import tornado.gen import json import requests import traceback from cesium_app.config import cfg IP = '127.0.0.1' PORT_SCHEDULER = 63500 if __name__ == "__main__": loop = tornado.ioloop.IOLoop.current() from distributed import Scheduler s = Scheduler(loop=loop) s.start(PORT_SCHEDULER) print('Task scheduler listening on port {}'.format(PORT_SCHEDULER)) from distributed import Worker w = Worker('127.0.0.1', PORT_SCHEDULER, loop=loop, ncores=1) w.start(0) print('Single worker activated') print('Starting main loop...') loop.start()
def test_add_remove_worker(s): events = [] class MyPlugin(SchedulerPlugin): def add_worker(self, worker, scheduler): assert scheduler is s events.append(('add_worker', worker)) def remove_worker(self, worker, scheduler): assert scheduler is s events.append(('remove_worker', worker)) plugin = MyPlugin() s.add_plugin(plugin) assert events == [] a = Worker(s.address) b = Worker(s.address) yield a._start() yield b._start() yield a._close() yield b._close() assert events == [('add_worker', a.address), ('add_worker', b.address), ('remove_worker', a.address), ('remove_worker', b.address), ] events[:] = [] s.remove_plugin(plugin) a = Worker(s.address) yield a._start() yield a._close() assert events == []
def test_scheduler_sees_memory_limits(s): w = yield Worker(s.address, nthreads=3, memory_limit=12345) assert s.workers[w.address].memory_limit == 12345 yield w.close()
def _run(cls, worker_args, worker_kwargs, worker_start_args, silence_logs, init_result_q, child_stop_q): # pragma: no cover from distributed import Worker try: from dask.multiprocessing import initialize_worker_process except ImportError: # old Dask version pass else: initialize_worker_process() if silence_logs: logger.setLevel(silence_logs) IOLoop.clear_instance() loop = IOLoop() loop.make_current() worker = Worker(*worker_args, **worker_kwargs) @gen.coroutine def do_stop(timeout): try: yield worker._close(report=False, nanny=False) finally: loop.stop() def watch_stop_q(): """ Wait for an incoming stop message and then stop the worker cleanly. """ while True: try: msg = child_stop_q.get(timeout=1000) except Empty: pass else: assert msg['op'] == 'stop' loop.add_callback(do_stop, msg['timeout']) break t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch") t.daemon = True t.start() @gen.coroutine def run(): """ Try to start worker and inform parent of outcome. """ try: yield worker._start(*worker_start_args) except Exception as e: logger.exception(e) init_result_q.put(e) else: assert worker.address init_result_q.put({'address': worker.address, 'dir': worker.local_dir}) yield worker.wait_until_closed() logger.info("Worker closed") try: loop.run_sync(run) except TimeoutError: # Loop was stopped before wait_until_closed() returned, ignore pass except KeyboardInterrupt: pass
async def test_simple(cleanup): async with Scheduler(protocol="ucx") as s: async with Worker(s.address) as a: async with Client(s.address, asynchronous=True) as c: result = await c.submit(lambda x: x + 1, 10) assert result == 11
async def test_deprecated_params(s, name): with pytest.warns(FutureWarning, match=name): async with Worker(s.address, **{name: 0.789}) as a: assert getattr(a.memory_manager, name) == 0.789
def test_scheduler_sees_memory_limits(s): w = Worker(s.ip, s.port, ncores=3, memory_limit=12345) yield w._start(0) assert s.worker_info[w.address]['memory_limit'] == 12345 yield w._close()
async def test_large_transfer(cleanup): np = pytest.importorskip("numpy") async with Scheduler(protocol="ws://") as s: async with Worker(s.address, protocol="ws://") as w: async with Client(s.address, asynchronous=True) as c: future = await c.scatter(np.random.random(1000000))
SCHEDULER_PORT = 5678 SCHEDULER_HTTP_PORT = 9786 SCHEDULER_BOKEH_PORT = 12345 SCHEDULER_IP = '127.0.0.1' HOME_PAGE = 'http://localhost:5050' print('Hello') loop_worker = IOLoop() t = Thread(target=loop_worker.start) t.start() NUM_WORKERS = 2 nanny_process = [ Worker(center_ip=SCHEDULER_IP, center_port=SCHEDULER_PORT, loop=loop_worker, ncores=2) for _ in range(NUM_WORKERS) ] for nanny in nanny_process: nanny.start() t.join() for nanny in nanny_process: nanny.start() loop_worker.stop()
from distributed import Worker #from distributed import Nanny from tornado.ioloop import IOLoop from threading import Thread w = Worker('tcp://dscheduler:8786') #w = Nanny('tcp://dscheduler:8786') w.start() # choose randomly assigned port loop = IOLoop.current() loop.start()