def g(): s = Scheduler(ip='127.0.0.1') done = s.start() s.listen(0) a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1') yield a._start() b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip) yield b._start() start = time() try: while len(s.ncores) < 2: yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") yield f(s, a, b) finally: logger.debug("Closing out test cluster") for w in [a, b]: with ignoring(TimeoutError, StreamClosedError, OSError): yield w._close() if os.path.exists(w.local_dir): shutil.rmtree(w.local_dir) yield s.close()
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist): given_host = host host = host or get_ip() ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) if _bokeh: try: import bokeh import distributed.bokeh hosts = ['%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname(), host] + list(bokeh_whitelist)] dirname = os.path.dirname(distributed.__file__) paths = [os.path.join(dirname, 'bokeh', name) for name in ['status', 'tasks']] binname = sys.argv[0][:-len('dscheduler')] + 'bokeh' args = ([binname, 'serve'] + paths + ['--log-level', 'warning', '--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port)] + sum([['--host', host] for host in hosts], [])) if show: args.append('--show') bokeh_options = {'host': host if given_host else '127.0.0.1', 'http-port': http_port, 'tcp-port': port, 'bokeh-port': bokeh_port} with open('.dask-web-ui.json', 'w') as f: json.dump(bokeh_options, f, indent=2) if sys.version_info[0] >= 3: from bokeh.command.bootstrap import main ctx = multiprocessing.get_context('spawn') bokeh_proc = ctx.Process(target=main, args=(args,)) bokeh_proc.daemon = True bokeh_proc.start() else: bokeh_proc = subprocess.Popen(args) logger.info(" Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() bokeh_proc.terminate() logger.info("End scheduler at %s:%d", ip, port)
def start_cluster(ncores, scheduler_addr, loop, security=None, Worker=Worker, scheduler_kwargs={}, worker_kwargs={}): s = Scheduler(loop=loop, validate=True, security=security, **scheduler_kwargs) done = s.start(scheduler_addr) workers = [Worker(s.address, ncores=ncore[1], name=i, security=security, loop=loop, validate=True, **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)) for i, ncore in enumerate(ncores)] for w in workers: w.rpc = workers[0].rpc yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)] start = time() while (len(s.workers) < len(ncores) or any(comm.comm is None for comm in s.stream_comms.values())): yield gen.sleep(0.01) if time() - start > 5: yield [w._close(timeout=1) for w in workers] yield s.close(fast=True) raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist): given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}, loop=loop) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(center, host, port, http_port, bokeh_port): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) try: import bokeh except ImportError: pass else: import distributed.diagnostics.bokeh hosts = ['%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname()]] dirname = os.path.dirname(distributed.__file__) path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status') proc = subprocess.Popen(['bokeh', 'serve', path, '--log-level', 'warning', '--port', str(bokeh_port)] + sum([['--host', host] for host in hosts], [])) distributed.diagnostics.bokeh.server_process = proc # monkey patch logger.info(" Start Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) loop.start() loop.close() scheduler.stop() logger.info("End scheduler at %s:%d", ip, port)
def test_scheduler_file(): with tmpfile() as fn: s = Scheduler(scheduler_file=fn) s.start(8008) w = Nanny(scheduler_file=fn) yield w._start() assert set(s.workers) == {w.worker_address} yield w._close() s.stop()
def main(scheduler_file, interface, nthreads, local_directory, memory_limit, scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port): if interface: host = get_ip_interface(interface) else: host = None if rank == 0 and scheduler: try: from distributed.bokeh.scheduler import BokehScheduler except ImportError: services = {} else: services = {('bokeh', bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)} scheduler = Scheduler(scheduler_file=scheduler_file, loop=loop, services=services) addr = uri_from_host_port(host, None, 8786) scheduler.start(addr) try: loop.start() loop.close() finally: scheduler.stop() else: W = Nanny if nanny else Worker worker = W(scheduler_file=scheduler_file, loop=loop, name=rank if scheduler else None, ncores=nthreads, local_dir=local_directory, services={('bokeh', bokeh_worker_port): BokehWorker}, memory_limit=memory_limit) addr = uri_from_host_port(host, None, 0) @gen.coroutine def run(): yield worker._start(addr) while worker.status != 'closed': yield gen.sleep(0.2) try: loop.run_sync(run) loop.close() finally: pass @gen.coroutine def close(): yield worker._close(timeout=2) loop.run_sync(close)
def main(center, port): loop = IOLoop.current() scheduler = Scheduler(center, services={'http': HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) done = scheduler.start(port) loop.start() loop.close() scheduler.stop() logging.info("End scheduler at %s:%d", ip, port)
def start_cluster(ncores, Worker=Worker): s = Scheduler(ip='127.0.0.1') done = s.start(0) workers = [Worker(s.ip, s.port, ncores=v, ip=k) for k, v in ncores] yield [w._start() for w in workers] start = time() while len(s.ncores) < len(ncores): yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def main(center, host, port): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, services={'http': HTTPScheduler}, ip=ip) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) loop.start() loop.close() scheduler.stop() logging.info("End scheduler at %s:%d", ip, port)
def start_cluster(ncores): s = Scheduler(ip='127.0.0.1') s.listen(0) workers = [Worker(s.ip, s.port, ncores=v, ip=k) for k, v in ncores] IOLoop.current().add_callback(s.start) yield [w._start() for w in workers] start = time() while len(s.ncores) < len(ncores): yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def start_cluster(ncores, loop, Worker=Worker): s = Scheduler(ip='127.0.0.1', loop=loop) done = s.start(0) workers = [Worker(s.ip, s.port, ncores=v, ip=k, name=i, loop=loop) for i, (k, v) in enumerate(ncores)] yield [w._start() for w in workers] start = time() while len(s.ncores) < len(ncores): yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def f(c, a, b): s = Scheduler((c.ip, c.port), loop=loop) yield s.sync_center() done = s.start(0) progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'finished' check_bar_completed(capsys) s.close() yield done
def run_scheduler(q, nputs, **kwargs): from distributed import Scheduler # On Python 2.7 and Unix, fork() is used to spawn child processes, # so avoid inheriting the parent's IO loop. with pristine_loop() as loop: scheduler = Scheduler(validate=True, **kwargs) done = scheduler.start('127.0.0.1') for i in range(nputs): q.put(scheduler.address) try: loop.start() finally: loop.close(all_fds=True)
def run_scheduler(q, scheduler_port=0, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() scheduler = Scheduler(loop=loop, validate=True, **kwargs) done = scheduler.start(scheduler_port) q.put(scheduler.port) try: loop.start() finally: loop.close(all_fds=True)
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(ip=ip, loop=loop, services={('http', http_port): HTTPScheduler}) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def f(): s = Scheduler(loop=loop) done = s.start(0) a = Worker(s.ip, s.port, loop=loop, ncores=1) b = Worker(s.ip, s.port, loop=loop, ncores=1) yield [a._start(0), b._start(0)] progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'finished' check_bar_completed(capsys) yield [a._close(), b._close()] s.close() yield done
def run_scheduler(q, scheduler_port=0, center_port=None, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = ('127.0.0.1', center_port) if center_port else None scheduler = Scheduler(center=center, loop=loop, validate=True, **kwargs) done = scheduler.start(scheduler_port) q.put(scheduler.port) try: loop.start() finally: loop.close(all_fds=True)
def main(center, host, port, http_port, bokeh_port, show, _bokeh): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) if _bokeh: try: import bokeh import distributed.bokeh hosts = ['%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname(), host]] dirname = os.path.dirname(distributed.__file__) paths = [os.path.join(dirname, 'bokeh', name) for name in ['status', 'tasks']] args = (['bokeh', 'serve'] + paths + ['--log-level', 'warning', '--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port)] + sum([['--host', host] for host in hosts], [])) if show: args.append('--show') from bokeh.command.bootstrap import main proc = multiprocessing.Process(target=main, args=(args,)) proc.daemon = True proc.start() logger.info(" Start Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() proc.terminate() logger.info("End scheduler at %s:%d", ip, port)
def run_scheduler(q, center_port=None, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = ('127.0.0.1', center_port) if center_port else None scheduler = Scheduler(center=center, **kwargs) scheduler.listen(0) if center_port: loop.run_sync(scheduler.sync_center) done = scheduler.start(0) q.put(scheduler.port) loop.start()
def start_cluster(ncores, loop, Worker=Worker, scheduler_kwargs={}, worker_kwargs={}): s = Scheduler(ip='127.0.0.1', loop=loop, validate=True, **scheduler_kwargs) done = s.start(0) workers = [Worker(s.ip, s.port, ncores=ncore[1], ip=ncore[0], name=i, loop=loop, validate=True, **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)) for i, ncore in enumerate(ncores)] for w in workers: w.rpc = workers[0].rpc yield [w._start() for w in workers] start = time() while len(s.ncores) < len(ncores): yield gen.sleep(0.01) if time() - start > 5: raise Exception("Cluster creation timeout") raise gen.Return((s, workers))
def f(c, a, b): s = Scheduler((c.ip, c.port), loop=loop) s.listen(0) yield s.sync_center() done = s.start() s.update_graph(dsk={'x': (div, 1, 0)}, keys=['x']) progress = TextProgressBar(['x'], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'error' assert progress.stream.closed() progress = TextProgressBar(['x'], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == 'error' assert progress.stream.closed() s.close() yield done
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() logger.info('-' * 47) services = {('http', http_port): HTTPScheduler} if _bokeh: from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', 8788)] = BokehScheduler scheduler = Scheduler(ip=ip, loop=loop, services=services) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info('-' * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
async def test_web_preload_worker(cleanup, worker_preload): async with Scheduler(port=8786, host="localhost") as s: async with Nanny( preload_nanny=["http://127.0.0.1:12346/preload"]) as nanny: assert nanny.scheduler_addr == s.address
def main( host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key, dashboard_address, ): g0, g1, g2 = gc.get_threshold() # https://github.com/dask/distributed/issues/1653 gc.set_threshold(g0 * 3, g1 * 3, g2 * 3) enable_proctitle_on_current() enable_proctitle_on_children() if bokeh_port is not None: warnings.warn( "The --bokeh-port flag has been renamed to --dashboard-address. " "Consider adding ``--dashboard-address :%d`` " % bokeh_port ) dashboard_address = bokeh_port sec = Security( tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key ) if not host and (tls_ca_file or tls_cert or tls_key): host = "tls://" if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix="scheduler-") local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info("-" * 47) services = {} if _bokeh: try: from distributed.bokeh.scheduler import BokehScheduler services[("bokeh", dashboard_address)] = ( BokehScheduler, {"prefix": bokeh_prefix}, ) except ImportError as error: if str(error).startswith("No module named"): logger.info("Web dashboard not loaded. Unable to import bokeh") else: logger.info("Unable to import bokeh: %s" % str(error)) scheduler = Scheduler( loop=loop, services=services, scheduler_file=scheduler_file, security=sec ) scheduler.start(addr) if not preload: preload = dask.config.get("distributed.scheduler.preload") if not preload_argv: preload_argv = dask.config.get("distributed.scheduler.preload-argv") preload_modules( preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv ) logger.info("Local Directory: %26s", local_directory) logger.info("-" * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key): enable_proctitle_on_current() enable_proctitle_on_children() sec = Security(tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key, ) if not host and (tls_ca_file or tls_cert or tls_key): host = 'tls://' if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix='scheduler-') local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {} if _bokeh: try: from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_port)] = (BokehScheduler, {'prefix': bokeh_prefix}) except ImportError as error: if str(error).startswith('No module named'): logger.info('Web dashboard not loaded. Unable to import bokeh') else: logger.info('Unable to import bokeh: %s' % str(error)) scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file, security=sec) scheduler.start(addr) if not preload: preload = dask.config.get('distributed.scheduler.preload') if not preload_argv: preload_argv = dask.config.get('distributed.scheduler.preload-argv') preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv) logger.info('Local Directory: %26s', local_directory) logger.info('-' * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def test_services(): s = Scheduler(services={'http': HTTPScheduler}) assert isinstance(s.services['http'], HTTPServer) assert s.services['http'].port
def test_services_with_port(): s = Scheduler(services={('http', 9999): HTTPScheduler}) assert isinstance(s.services['http'], HTTPServer) assert s.services['http'].port == 9999
async def test_tls_scheduler(security, cleanup): async with Scheduler(security=security, host="localhost") as s: assert s.address.startswith("tls")
async def test_config(cleanup): async with Scheduler() as s: async with Nanny(s.address, config={"foo": "bar"}) as n: async with Client(s.address, asynchronous=True) as client: config = await client.run(dask.config.get, "foo") assert config[n.worker_address] == "bar"
def _create_scheduler(self): scheduler = Scheduler(ip=self.ip) scheduler.start(self.port) return scheduler
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ":" in host and port == 8786: host, port = host.rsplit(":", 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() logger.info("-" * 47) services = {("http", http_port): HTTPScheduler} if _bokeh: from distributed.bokeh.scheduler import BokehScheduler services[("bokeh", 8788)] = BokehScheduler scheduler = Scheduler(ip=ip, loop=loop, services=services) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface( host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False, ) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info("-" * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key): enable_proctitle_on_current() enable_proctitle_on_children() sec = Security( tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key, ) if not host and (tls_ca_file or tls_cert or tls_key): host = 'tls://' if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix='scheduler-') local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {} if _bokeh: with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_port)] = (BokehScheduler, { 'prefix': bokeh_prefix }) scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file, security=sec) scheduler.start(addr) preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv) logger.info('Local Directory: %26s', local_directory) logger.info('-' * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def test_tls_scheduler(security, loop): s = Scheduler(security=security, loop=loop) s.start('localhost') assert s.address.startswith('tls') s.close()
def main( host, port, bokeh_port, show, dashboard, bokeh, dashboard_prefix, use_xheaders, pid_file, tls_ca_file, tls_cert, tls_key, dashboard_address, **kwargs ): g0, g1, g2 = gc.get_threshold() # https://github.com/dask/distributed/issues/1653 gc.set_threshold(g0 * 3, g1 * 3, g2 * 3) enable_proctitle_on_current() enable_proctitle_on_children() if bokeh_port is not None: warnings.warn( "The --bokeh-port flag has been renamed to --dashboard-address. " "Consider adding ``--dashboard-address :%d`` " % bokeh_port ) dashboard_address = bokeh_port if bokeh is not None: warnings.warn( "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. " ) dashboard = bokeh if port is None and (not host or not re.search(r":\d", host)): port = 8786 sec = Security( **{ k: v for k, v in [ ("tls_ca_file", tls_ca_file), ("tls_scheduler_cert", tls_cert), ("tls_scheduler_key", tls_key), ] if v is not None } ) if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ: config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"]) # Update the global config given priority to the existing global config dask.config.update(dask.config.global_config, config, priority="old") if not host and (tls_ca_file or tls_cert or tls_key): host = "tls://" if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) loop = IOLoop.current() logger.info("-" * 47) scheduler = Scheduler( loop=loop, security=sec, host=host, port=port, dashboard_address=dashboard_address if dashboard else None, service_kwargs={"dashboard": {"prefix": dashboard_prefix}}, **kwargs ) logger.info("-" * 47) install_signal_handlers(loop) async def run(): await scheduler await scheduler.finished() try: loop.run_sync(run) finally: scheduler.stop() logger.info("End scheduler at %r", scheduler.address)