def f(c, a, b): s = Scheduler((c.ip, c.port), loop=loop) s.listen(0) yield s.sync_center() done = s.start() s.update_graph(dsk={'x': (div, 1, 0)}, keys=['x']) progress = TextProgressBar(['x'], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'error' assert progress.stream.closed() progress = TextProgressBar(['x'], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'error' assert progress.stream.closed() s.close() yield done
def g(): s = Scheduler(ip='127.0.0.1') done = s.start() s.listen(0) a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(s.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(s, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) yield s.close()
def start_cluster(ncores): s = Scheduler(ip='127.0.0.1') s.listen(0) workers = [Worker(s.ip, s.port, ncores=v, ip=k) for k, v in ncores] IOLoop.current().add_callback(s.start) yield [w._start() for w in workers] start = time() while len(s.ncores) < len(ncores): yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def f(c, a, b): s = Scheduler((c.ip, c.port), loop=loop) s.listen(0) yield s.sync_center() done = s.start() progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'finished' check_bar_completed(capsys) s.close() yield done
def run_scheduler(q, center_port=None, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = ('127.0.0.1', center_port) if center_port else None scheduler = Scheduler(center=center, **kwargs) scheduler.listen(0) done = scheduler.start(0) q.put(scheduler.port) try: loop.start() finally: loop.close(all_fds=True)
def run_scheduler(q, center_port=None, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = ('127.0.0.1', center_port) if center_port else None scheduler = Scheduler(center=center, **kwargs) scheduler.listen(0) if center_port: loop.run_sync(scheduler.sync_center) done = scheduler.start(0) q.put(scheduler.port) loop.start()