def g(): c = Center('127.0.0.1') c.listen(0) a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker(c.ip, c.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) except Exception as e: logger.exception(e) raise finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) c.stop()
def g(): s = Scheduler(ip='127.0.0.1') done = s.start() s.listen(0) a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(s.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(s, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) yield s.close()
def test_coerce_address(): with dask.config.set({'distributed.comm.timeouts.connect': '100ms'}): s = Scheduler(validate=True) s.start(0) print("scheduler:", s.address, s.listen_address) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker('127.0.0.1', s.port, name='charlie') yield [a._start(), b._start(), c._start()] assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(a.address) == a.address # Aliases assert s.coerce_address('alice') == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address assert s.coerce_hostname('127.0.0.1') == '127.0.0.1' assert s.coerce_hostname('alice') == a.ip assert s.coerce_hostname(123) == b.ip assert s.coerce_hostname('charlie') == c.ip assert s.coerce_hostname('jimmy') == 'jimmy' assert s.coerce_address('zzzt:8000', resolve=False) == 'tcp://zzzt:8000' yield s.close() yield [w._close() for w in [a, b, c]]
def test_add_remove_worker(s): events = [] class MyPlugin(SchedulerPlugin): def add_worker(self, worker, scheduler): assert scheduler is s events.append(('add_worker', worker)) def remove_worker(self, worker, scheduler): assert scheduler is s events.append(('remove_worker', worker)) plugin = MyPlugin() s.add_plugin(plugin) assert events == [] a = Worker(s.address) b = Worker(s.address) yield a._start() yield b._start() yield a._close() yield b._close() assert events == [('add_worker', a.address), ('add_worker', b.address), ('remove_worker', a.address), ('remove_worker', b.address), ] events[:] = [] s.remove_plugin(plugin) a = Worker(s.address) yield a._start() yield a._close() assert events == []
def test_coerce_address(): s = Scheduler(validate=True) s.start(0) print("scheduler:", s.address, s.listen_address) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker('127.0.0.1', s.port, name='charlie') yield [a._start(), b._start(), c._start()] assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000' assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000' assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000') assert s.coerce_address(a.address) == a.address # Aliases assert s.coerce_address('alice') == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address assert s.coerce_hostname('127.0.0.1') == '127.0.0.1' assert s.coerce_hostname('alice') == a.ip assert s.coerce_hostname(123) == b.ip assert s.coerce_hostname('charlie') == c.ip assert s.coerce_hostname('jimmy') == 'jimmy' assert s.coerce_address('zzzt:8000', resolve=False) == 'tcp://zzzt:8000' yield s.close() yield [w._close() for w in [a, b, c]]
def test_resources(c, s): assert not s.worker_resources assert not s.resources a = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 2}) b = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 1, 'DB': 1}) yield [a._start(), b._start()] assert s.resources == { 'GPU': { a.address: 2, b.address: 1 }, 'DB': { b.address: 1 } } assert s.worker_resources == { a.address: { 'GPU': 2 }, b.address: { 'GPU': 1, 'DB': 1 } } yield b._close() assert s.resources == {'GPU': {a.address: 2}, 'DB': {}} assert s.worker_resources == {a.address: {'GPU': 2}} yield a._close()
def g(): s = Scheduler(ip='127.0.0.1') done = s.start() s.listen(0) a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(s.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(s, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) yield s.close()
def g(): c = Center('127.0.0.1') c.listen(0) a = Worker(c.ip, c.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker(c.ip, c.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) except Exception as e: logger.exception(e) raise finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) c.stop()
def test_worker_name(): s = Scheduler(validate=True) s.start(0) w = Worker(s.ip, s.port, name='alice') yield w._start() assert s.worker_info[w.address]['name'] == 'alice' assert s.aliases['alice'] == w.address with pytest.raises(ValueError): w = Worker(s.ip, s.port, name='alice') yield w._start() yield s.close() yield w._close()
def test_worker_name(): s = Scheduler() s.start(0) w = Worker(s.ip, s.port, name='alice') yield w._start() assert s.worker_info[w.address]['name'] == 'alice' assert s.aliases['alice'] == w.address with pytest.raises(ValueError): w = Worker(s.ip, s.port, name='alice') yield w._start() yield s.close() yield w._close()
def test_worker_name(): s = Scheduler(validate=True) s.start(0) w = Worker(s.ip, s.port, name='alice') yield w._start() assert s.workers[w.address].name == 'alice' assert s.aliases['alice'] == w.address with pytest.raises(ValueError): w2 = Worker(s.ip, s.port, name='alice') yield w2._start() yield w2._close() yield s.close() yield w._close()
def test_no_workers_to_memory(c, s): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while not s.workers: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def test_services(s, a, b): c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1', services={'http': HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert c.service_ports['http'] == c.services['http'].port assert s.worker_services[c.address]['http'] == c.service_ports['http']
def test_no_workers_to_memory(c, s): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while not s.workers: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def test_no_workers_to_memory(c, s): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not s.task_state: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while not s.workers: yield gen.sleep(0.01) assert s.task_state[y.key] == 'memory' assert s.task_state[x.key] == 'released' assert s.task_state[z.key] == 'processing' yield w._close()
def test_worker_arrives_with_processing_data(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not any(w.processing for w in s.workers.values()): yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while len(s.workers) < 3: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def test_no_worker_to_memory_restrictions(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'}) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1, name='alice') w.put_key_in_memory(y.key, 3) yield w._start() while len(s.workers) < 3: yield gen.sleep(0.01) yield gen.sleep(0.3) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def test_worker_arrives_with_processing_data(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not any(w.processing for w in s.workers.values()): yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1) w.put_key_in_memory(y.key, 3) yield w._start() start = time() while len(s.workers) < 3: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def test_no_worker_to_memory_restrictions(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'}) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1, name='alice') w.put_key_in_memory(y.key, 3) yield w._start() while len(s.workers) < 3: yield gen.sleep(0.01) yield gen.sleep(0.3) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: 'released', y.key: 'memory', z.key: 'processing', } yield w._close()
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services, coerce_address=False) for k in w.data: assert w.address in s.who_has[k] s.validate() assert w.ip in s.host_info assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port]))
def test_worker_arrives_with_processing_data(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not s.processing: yield gen.sleep(0.01) w = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1') w.put_key_in_memory(y.key, 3) yield w._start() start = time() while len(s.workers) < 3: yield gen.sleep(0.01) assert s.task_state[y.key] == 'memory' assert s.task_state[x.key] == 'released' assert s.task_state[z.key] == 'processing' yield w._close()
def f(): s = Scheduler(loop=loop) done = s.start(0) a = Worker(s.ip, s.port, loop=loop, ncores=1) b = Worker(s.ip, s.port, loop=loop, ncores=1) yield [a._start(0), b._start(0)] progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'finished' check_bar_completed(capsys) yield [a._close(), b._close()] s.close() yield done
def test_scatter_no_workers(s): with pytest.raises(gen.TimeoutError): yield gen.with_timeout(timedelta(seconds=0.1), s.scatter(data={'x': dumps(1)}, client='alice')) w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') yield [s.scatter(data={'x': dumps(1)}, client='alice'), w._start()] assert w.data['x'] == 1
def test_scatter_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield s.scatter(data={'x': 1}, client='alice', timeout=0.1) w = Worker(s.ip, s.port, ncores=3) yield [c._scatter(data={'x': 1}), w._start()] assert w.data['x'] == 1 yield w._close()
def test_services(s, a, b): c = Worker(s.ip, s.port, ncores=1, services={'http': HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert c.service_ports['http'] == c.services['http'].port assert s.worker_info[ c.address]['services']['http'] == c.service_ports['http'] yield c._close()
def test_services_port(s, a, b): c = Worker(s.ip, s.port, ncores=1, services={('http', 9898): HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert (c.service_ports['http'] == c.services['http'].port == s.worker_info[c.address]['services']['http'] == 9898) c.services['http'].stop() yield c._close()
def test_scatter_no_workers(s): with pytest.raises(gen.TimeoutError): yield gen.with_timeout(timedelta(seconds=0.1), s.scatter(data={'x': dumps(1)}, client='alice')) w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') yield [s.scatter(data={'x': dumps(1)}, client='alice'), w._start()] assert w.data['x'] == 1
def g(): c = Center('127.0.0.1', 8017) c.listen(c.port) a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=2) yield a._start() b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1) yield b._start() while len(c.ncores) < 2: yield gen.sleep(0.01) try: yield f(c, a, b) finally: with ignoring(): yield a._close() with ignoring(): yield b._close() c.stop()
def g(): c = Center('127.0.0.1', 8017) c.listen(c.port) a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=2) yield a._start() b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1) yield b._start() while len(c.ncores) < 2: yield gen.sleep(0.01) try: yield f(c, a, b) finally: with ignoring(): yield a._close() with ignoring(): yield b._close() c.stop()
def test_create_with_client(c, s): yield c.register_worker_plugin(MyPlugin(123)) worker = Worker(s.address, loop=s.loop) yield worker._start() assert worker._my_plugin_status == "setup" assert worker._my_plugin_data == 123 yield worker._close() assert worker._my_plugin_status == "teardown"
def test_resources(c, s): assert not s.worker_resources assert not s.resources a = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 2}) b = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 1, 'DB': 1}) yield [a._start(), b._start()] assert s.resources == {'GPU': {a.address: 2, b.address: 1}, 'DB': {b.address: 1}} assert s.worker_resources == {a.address: {'GPU': 2}, b.address: {'GPU': 1, 'DB': 1}} yield b._close() assert s.resources == {'GPU': {a.address: 2}, 'DB': {}} assert s.worker_resources == {a.address: {'GPU': 2}} yield a._close()
def test_services_port(s, a, b): c = Worker(s.ip, s.port, ncores=1, ip='127.0.0.1', services={('http', 9898): HTTPWorker}) yield c._start() assert isinstance(c.services['http'], HTTPServer) assert (c.service_ports['http'] == c.services['http'].port == s.worker_info[c.address]['services']['http'] == 9898) c.services['http'].stop() yield c._close()
def run_worker(q, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) loop.start()
def test_scatter_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield s.scatter(data={"x": 1}, client="alice", timeout=0.1) start = time() with pytest.raises(gen.TimeoutError): yield c.scatter(123, timeout=0.1) assert time() < start + 1.5 w = Worker(s.ip, s.port, ncores=3) yield [c.scatter(data={"y": 2}, timeout=5), w._start()] assert w.data["y"] == 2 yield w._close()
def g(): c = Center("127.0.0.1", 8017) c.listen(c.port) a = Worker("127.0.0.2", 8018, c.ip, c.port, ncores=2) yield a._start() b = Worker("127.0.0.3", 8019, c.ip, c.port, ncores=1) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) finally: logger.debug("Closing out test cluster") with ignoring(): yield a._close() with ignoring(): yield b._close() c.stop()
def test_scatter_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield s.scatter(data={'x': 1}, client='alice', timeout=0.1) start = time() with pytest.raises(gen.TimeoutError): yield c.scatter(123, timeout=0.1) assert time() < start + 1.5 w = Worker(s.ip, s.port, ncores=3) yield [c.scatter(data={'y': 2}, timeout=5), w._start()] assert w.data['y'] == 2 yield w._close()
def test_coerce_address(): s = Scheduler() s.start(0) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker(s.ip, s.port, name='charlie', ip='127.0.0.2') yield [a._start(), b._start(), c._start()] assert s.coerce_address(b'127.0.0.1') == '127.0.0.1' assert s.coerce_address(('127.0.0.1', 8000)) == '127.0.0.1:8000' assert s.coerce_address(['127.0.0.1', 8000]) == '127.0.0.1:8000' assert s.coerce_address([b'127.0.0.1', 8000]) == '127.0.0.1:8000' assert s.coerce_address(('127.0.0.1', '8000')) == '127.0.0.1:8000' assert s.coerce_address(b'localhost') == '127.0.0.1' assert s.coerce_address('localhost') == '127.0.0.1' assert s.coerce_address(u'localhost') == '127.0.0.1' assert s.coerce_address('localhost:8000') == '127.0.0.1:8000' assert s.coerce_address(a.address) == a.address assert s.coerce_address(a.address_tuple) == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address yield s.close() yield [w._close() for w in [a, b, c]]
def g(): c = Center('127.0.0.1', 8017) c.listen(c.port) a = Worker('127.0.0.2', 8018, c.ip, c.port, ncores=2) yield a._start() b = Worker('127.0.0.3', 8019, c.ip, c.port, ncores=1) yield b._start() start = time() try: while len(c.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(c, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) c.stop()
def run_worker(q, scheduler_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1', loop=loop, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.close(all_fds=True)
def test_coerce_address(): s = Scheduler(validate=True) s.start(0) a = Worker(s.ip, s.port, name='alice') b = Worker(s.ip, s.port, name=123) c = Worker(s.ip, s.port, name='charlie', ip='127.0.0.2') yield [a._start(), b._start(), c._start()] assert s.coerce_address(b'127.0.0.1') == '127.0.0.1' assert s.coerce_address(('127.0.0.1', 8000)) == '127.0.0.1:8000' assert s.coerce_address(['127.0.0.1', 8000]) == '127.0.0.1:8000' assert s.coerce_address([b'127.0.0.1', 8000]) == '127.0.0.1:8000' assert s.coerce_address(('127.0.0.1', '8000')) == '127.0.0.1:8000' assert s.coerce_address(b'localhost') == '127.0.0.1' assert s.coerce_address('localhost') == '127.0.0.1' assert s.coerce_address(u'localhost') == '127.0.0.1' assert s.coerce_address('localhost:8000') == '127.0.0.1:8000' assert s.coerce_address(a.address) == a.address assert s.coerce_address(a.address_tuple) == a.address assert s.coerce_address(123) == b.address assert s.coerce_address('charlie') == c.address yield s.close() yield [w._close() for w in [a, b, c]]
def run_worker(q, scheduler_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1', loop=loop, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.close(all_fds=True)
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data[('x', 5)] = 6 w.data['y'] = 1 yield w._start(8020) dsk = {('x', i): (inc, i) for i in range(10)} s.update_graph(dsk=dsk, keys=list(dsk)) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, nanny_port=w.nanny_port) for k in w.data: assert w.address in s.who_has[k] s.validate(allow_overlap=True)
def test_file_descriptors_dont_leak(s): psutil = pytest.importorskip('psutil') proc = psutil.Process() before = proc.num_fds() w = Worker(s.ip, s.port) yield w._start(0) yield w._close() during = proc.num_fds() start = time() while proc.num_fds() > before: yield gen.sleep(0.01) assert time() < start + 5
def run_worker(q, scheduler_q, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback with log_errors(): with pristine_loop() as loop: PeriodicCallback(lambda: None, 500).start() scheduler_addr = scheduler_q.get() worker = Worker(scheduler_addr, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.address) try: loop.start() finally: loop.close(all_fds=True)
def test_file_descriptors_dont_leak(s): psutil = pytest.importorskip('psutil') proc = psutil.Process() before = proc.num_fds() w = Worker(s.ip, s.port) yield w._start(0) yield w._close() during = proc.num_fds() start = time() while proc.num_fds() > before: yield gen.sleep(0.01) assert time() < start + 5
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data[('x', 5)] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x', i): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services) for k in w.data: assert w.address in s.who_has[k] s.validate(allow_overlap=True)
def run_worker(q, scheduler_q, **kwargs): from distributed import Worker with log_errors(): with pristine_loop() as loop: scheduler_addr = scheduler_q.get() worker = Worker(scheduler_addr, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.address) try: @gen.coroutine def wait_until_closed(): yield worker._closed.wait() loop.run_sync(wait_until_closed) finally: loop.close(all_fds=True)
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i).encode(): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services, coerce_address=False) s.validate() assert w.ip in s.host_info assert s.host_info[w.ip]['ports'] == set(map(str, [a.port, b.port, w.port]))
def run_worker(q, scheduler_q, **kwargs): from distributed import Worker reset_logger_locks() with log_errors(): with pristine_loop() as loop: scheduler_addr = scheduler_q.get() worker = Worker(scheduler_addr, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.address) try: @gen.coroutine def wait_until_closed(): yield worker._closed.wait() loop.run_sync(wait_until_closed) finally: loop.close(all_fds=True)
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3) w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services) s.validate_state() assert w.ip in s.host_info assert s.host_info[w.ip]['addresses'] == {a.address, b.address, w.address} yield w._close()
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3) w.data['x-5'] = 6 w.data['y'] = 1 yield w._start(0) dsk = {('x-%d' % i): (inc, i) for i in range(10)} s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client', dependencies={k: set() for k in dsk}) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services) s.validate_state() assert w.ip in s.host_info assert s.host_info[w.ip]['addresses'] == {a.address, b.address, w.address} yield w._close()
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3, ip='127.0.0.1') w.data[('x', 5)] = 6 w.data['y'] = 1 yield w._start(8020) dsk = {('x', i): (inc, i) for i in range(10)} s.update_graph(dsk=dsk, keys=list(dsk)) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, nanny_port=w.nanny_port) for k in w.data: assert w.address in s.who_has[k] s.validate(allow_overlap=True)
def test_resource_submit(c, s, a, b): x = c.submit(inc, 1, resources={'A': 3}) y = c.submit(inc, 2, resources={'B': 1}) z = c.submit(inc, 3, resources={'C': 2}) yield wait(x) assert x.key in a.data yield wait(y) assert y.key in b.data assert s.get_task_status(keys=[z.key]) == {z.key: 'no-worker'} d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10}) yield d._start() yield wait(z) assert z.key in d.data yield d._close()
def test_new_worker_steals(c, s, a): yield wait(c.submit(slowinc, 1, delay=0.01)) futures = c.map(slowinc, range(100), delay=0.05) total = c.submit(sum, futures) while len(a.task_state) < 10: yield gen.sleep(0.01) b = Worker(s.ip, s.port, loop=s.loop, ncores=1, memory_limit=TOTAL_MEMORY) yield b._start() result = yield total assert result == sum(map(inc, range(100))) for w in [a, b]: assert all(isinstance(v, int) for v in w.data.values()) assert b.data yield b._close()
def test_new_worker_steals(c, s, a): yield _wait(c.submit(slowinc, 1, delay=0.01)._result()) futures = c.map(slowinc, range(100), delay=0.05) total = c.submit(sum, futures) while len(a.task_state) < 10: yield gen.sleep(0.01) b = Worker(s.ip, s.port, loop=s.loop, ncores=1) yield b._start() result = yield total._result() assert result == sum(map(inc, range(100))) for w in [a, b]: assert all(isinstance(v, int) for v in w.data.values()) assert b.data yield b._close()
def test_resource_submit(c, s, a, b): x = c.submit(inc, 1, resources={'A': 3}) y = c.submit(inc, 2, resources={'B': 1}) z = c.submit(inc, 3, resources={'C': 2}) yield _wait(x) assert x.key in a.data yield _wait(y) assert y.key in b.data assert z.key in s.unrunnable d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10}) yield d._start() yield _wait(z) assert z.key in d.data yield d._close()
def test_resource_submit(c, s, a, b): x = c.submit(inc, 1, resources={'A': 3}) y = c.submit(inc, 2, resources={'B': 1}) z = c.submit(inc, 3, resources={'C': 2}) yield _wait(x) assert x.key in a.data yield _wait(y) assert y.key in b.data assert z.key in s.unrunnable d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10}) yield d._start() yield _wait(z) assert z.key in d.data yield d._close()