def main(center, host, port, http_port, bokeh_port): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) try: import bokeh except ImportError: pass else: import distributed.diagnostics.bokeh hosts = ['%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname()]] dirname = os.path.dirname(distributed.__file__) path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status') proc = subprocess.Popen(['bokeh', 'serve', path, '--log-level', 'warning', '--port', str(bokeh_port)] + sum([['--host', host] for host in hosts], [])) distributed.diagnostics.bokeh.server_process = proc # monkey patch logger.info(" Start Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) loop.start() loop.close() scheduler.stop() logger.info("End scheduler at %s:%d", ip, port)
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist): given_host = host host = host or get_ip() ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) if _bokeh: try: import bokeh import distributed.bokeh hosts = ['%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname(), host] + list(bokeh_whitelist)] dirname = os.path.dirname(distributed.__file__) paths = [os.path.join(dirname, 'bokeh', name) for name in ['status', 'tasks']] binname = sys.argv[0][:-len('dscheduler')] + 'bokeh' args = ([binname, 'serve'] + paths + ['--log-level', 'warning', '--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port)] + sum([['--host', host] for host in hosts], [])) if show: args.append('--show') bokeh_options = {'host': host if given_host else '127.0.0.1', 'http-port': http_port, 'tcp-port': port, 'bokeh-port': bokeh_port} with open('.dask-web-ui.json', 'w') as f: json.dump(bokeh_options, f, indent=2) if sys.version_info[0] >= 3: from bokeh.command.bootstrap import main ctx = multiprocessing.get_context('spawn') bokeh_proc = ctx.Process(target=main, args=(args,)) bokeh_proc.daemon = True bokeh_proc.start() else: bokeh_proc = subprocess.Popen(args) logger.info(" Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() bokeh_proc.terminate() logger.info("End scheduler at %s:%d", ip, port)
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist): given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}, loop=loop) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(host, port, http_port, bokeh_port, bokeh_internal_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {('http', http_port): HTTPScheduler} if _bokeh: with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_internal_port)] = BokehScheduler scheduler = Scheduler(loop=loop, services=services) scheduler.start(addr) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(http_port=http_port, tcp_port=scheduler.port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info('-' * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %r", addr)
def test_scheduler_file(): with tmpfile() as fn: s = Scheduler(scheduler_file=fn) s.start(8008) w = yield Nanny(scheduler_file=fn) assert set(s.workers) == {w.worker_address} yield w.close() s.stop()
def test_scheduler_file(): with tmpfile() as fn: s = Scheduler(scheduler_file=fn) s.start(8008) w = Nanny(scheduler_file=fn) yield w._start() assert s.workers == {w.worker_address} yield w._close() s.stop()
def test_scheduler_file(): with tmpfile() as fn: s = Scheduler(scheduler_file=fn) s.start(8008) w = Nanny(scheduler_file=fn) yield w._start() assert set(s.workers) == {w.worker_address} yield w._close() s.stop()
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(ip=ip, loop=loop, services={('http', http_port): HTTPScheduler}) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(scheduler_file, interface, nthreads, local_directory, memory_limit, scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port): if interface: host = get_ip_interface(interface) else: host = None if rank == 0 and scheduler: try: from distributed.bokeh.scheduler import BokehScheduler except ImportError: services = {} else: services = { ('bokeh', bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix) } scheduler = Scheduler(scheduler_file=scheduler_file, loop=loop, services=services) addr = uri_from_host_port(host, None, 8786) scheduler.start(addr) try: loop.start() loop.close() finally: scheduler.stop() else: W = Nanny if nanny else Worker worker = W(scheduler_file=scheduler_file, loop=loop, name=rank if scheduler else None, ncores=nthreads, local_dir=local_directory, services={('bokeh', bokeh_worker_port): BokehWorker}, memory_limit=memory_limit) addr = uri_from_host_port(host, None, 0) @gen.coroutine def run(): yield worker._start(addr) while worker.status != 'closed': yield gen.sleep(0.2) try: loop.run_sync(run) loop.close() finally: pass @gen.coroutine def close(): yield worker._close(timeout=2) loop.run_sync(close)
def main(center, port): loop = IOLoop.current() scheduler = Scheduler(center, services={'http': HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) done = scheduler.start(port) loop.start() loop.close() scheduler.stop() logging.info("End scheduler at %s:%d", ip, port)
def main(scheduler_file, interface, nthreads, local_directory, memory_limit, scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port): if interface: host = get_ip_interface(interface) else: host = None if rank == 0 and scheduler: try: from distributed.bokeh.scheduler import BokehScheduler except ImportError: services = {} else: services = {('bokeh', bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)} scheduler = Scheduler(scheduler_file=scheduler_file, loop=loop, services=services) addr = uri_from_host_port(host, None, 8786) scheduler.start(addr) try: loop.start() loop.close() finally: scheduler.stop() else: W = Nanny if nanny else Worker worker = W(scheduler_file=scheduler_file, loop=loop, name=rank if scheduler else None, ncores=nthreads, local_dir=local_directory, services={('bokeh', bokeh_worker_port): BokehWorker}, memory_limit=memory_limit) addr = uri_from_host_port(host, None, 0) @gen.coroutine def run(): yield worker._start(addr) while worker.status != 'closed': yield gen.sleep(0.2) try: loop.run_sync(run) loop.close() finally: pass @gen.coroutine def close(): yield worker._close(timeout=2) loop.run_sync(close)
def main(center, host, port): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, services={'http': HTTPScheduler}, ip=ip) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) loop.start() loop.close() scheduler.stop() logging.info("End scheduler at %s:%d", ip, port)
def main(center, host, port, http_port, bokeh_port, show, _bokeh): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) if _bokeh: try: import bokeh import distributed.bokeh hosts = [ '%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname(), host] ] dirname = os.path.dirname(distributed.__file__) paths = [ os.path.join(dirname, 'bokeh', name) for name in ['status', 'tasks'] ] args = (['bokeh', 'serve'] + paths + [ '--log-level', 'warning', '--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port) ] + sum([['--host', host] for host in hosts], [])) if show: args.append('--show') from bokeh.command.bootstrap import main proc = multiprocessing.Process(target=main, args=(args, )) proc.daemon = True proc.start() logger.info(" Start Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() proc.terminate() logger.info("End scheduler at %s:%d", ip, port)
def scheduler(): # pragma: nocover app_client = skein.ApplicationClient.from_current() enable_proctitle_on_current() enable_proctitle_on_children() if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) addr = 'tcp://' loop = IOLoop.current() services = {} bokeh = False with ignoring(ImportError): try: from distributed.dashboard.scheduler import BokehScheduler except ImportError: # Old import location from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', 0)] = (BokehScheduler, {}) bokeh = True scheduler = Scheduler(loop=loop, services=services) scheduler.start(addr) install_signal_handlers(loop) # Set dask.dashboard before dask.scheduler since the YarnCluster object # waits on dask.scheduler only if bokeh: bokeh_port = scheduler.services['bokeh'].port bokeh_host = urlparse(scheduler.address).hostname bokeh_address = 'http://%s:%d' % (bokeh_host, bokeh_port) app_client.kv['dask.dashboard'] = bokeh_address.encode() app_client.kv['dask.scheduler'] = scheduler.address.encode() try: loop.start() loop.close() finally: scheduler.stop()
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(ip=ip, loop=loop, services={('http', http_port): HTTPScheduler}) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(scheduler_file, interface, nthreads, local_directory, memory_limit, scheduler): if interface: host = get_ip_interface(interface) else: host = None if rank == 0 and scheduler: scheduler = Scheduler(scheduler_file=scheduler_file, loop=loop, services={('bokeh', 8787): BokehScheduler}) addr = uri_from_host_port(host, None, 8786) scheduler.start(addr) try: loop.start() loop.close() finally: scheduler.stop() else: worker = Worker(scheduler_file=scheduler_file, loop=loop, name=rank if scheduler else None, ncores=nthreads, local_dir=local_directory, services={'bokeh': BokehWorker}, memory_limit=memory_limit) addr = uri_from_host_port(host, None, 0) @gen.coroutine def run(): yield worker._start(addr) while worker.status != 'closed': yield gen.sleep(0.2) try: loop.run_sync(run) loop.close() finally: pass @gen.coroutine def close(): yield worker._close(timeout=2) loop.run_sync(close)
def main(center, host, port, http_port, bokeh_port, show, _bokeh): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) if _bokeh: try: import bokeh import distributed.bokeh hosts = ['%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname(), host]] dirname = os.path.dirname(distributed.__file__) paths = [os.path.join(dirname, 'bokeh', name) for name in ['status', 'tasks']] args = (['bokeh', 'serve'] + paths + ['--log-level', 'warning', '--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port)] + sum([['--host', host] for host in hosts], [])) if show: args.append('--show') from bokeh.command.bootstrap import main proc = multiprocessing.Process(target=main, args=(args,)) proc.daemon = True proc.start() logger.info(" Start Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() proc.terminate() logger.info("End scheduler at %s:%d", ip, port)
def main(): app_client = skein.ApplicationClient.from_current() enable_proctitle_on_current() enable_proctitle_on_children() if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) addr = uri_from_host_port('', None, 0) loop = IOLoop.current() services = {} bokeh = False with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', 0)] = (BokehScheduler, {}) bokeh = True scheduler = Scheduler(loop=loop, services=services) scheduler.start(addr) install_signal_handlers(loop) app_client.kv['dask.scheduler'] = scheduler.address.encode() if bokeh: bokeh_port = scheduler.services['bokeh'].port bokeh_host = urlparse(scheduler.address).hostname bokeh_address = 'http://%s:%d' % (bokeh_host, bokeh_port) app_client.kv['dask.dashboard'] = bokeh_address.encode() try: loop.start() loop.close() finally: scheduler.stop()
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders): given_host = host host = host or get_ip() if ':' in host and port == 8786: host, port = host.rsplit(':', 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(ip=ip, loop=loop, services={('http', http_port): HTTPScheduler}) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(center, host, port, http_port, bokeh_port): ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) if center: loop.run_sync(scheduler.sync_center) scheduler.start(port) try: import bokeh except ImportError: pass else: import distributed.diagnostics.bokeh hosts = [ '%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname()] ] dirname = os.path.dirname(distributed.__file__) path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status') proc = subprocess.Popen([ 'bokeh', 'serve', path, '--log-level', 'warning', '--port', str(bokeh_port) ] + sum([['--host', host] for host in hosts], [])) distributed.diagnostics.bokeh.server_process = proc # monkey patch logger.info(" Start Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) loop.start() loop.close() scheduler.stop() logger.info("End scheduler at %s:%d", ip, port)
def scheduler(): # pragma: nocover app_client = skein.ApplicationClient.from_current() enable_proctitle_on_current() enable_proctitle_on_children() if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) loop = IOLoop.current() scheduler = Scheduler(loop=loop, dashboard_address=("", 0)) install_signal_handlers(loop) def post_addresses(): # Set dask.dashboard before dask.scheduler since the YarnCluster object # waits on dask.scheduler only if "dashboard" in scheduler.services: bokeh_port = scheduler.services["dashboard"].port bokeh_host = urlparse(scheduler.address).hostname bokeh_address = "http://%s:%d" % (bokeh_host, bokeh_port) app_client.kv["dask.dashboard"] = bokeh_address.encode() app_client.kv["dask.scheduler"] = scheduler.address.encode() async def run(): await scheduler await loop.run_in_executor(None, post_addresses) await scheduler.finished() try: loop.run_sync(run) except (KeyboardInterrupt, TimeoutError): pass finally: scheduler.stop()
def main( host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key, dashboard_address, ): enable_proctitle_on_current() enable_proctitle_on_children() if bokeh_port is not None: warnings.warn( "The --bokeh-port flag has been renamed to --dashboard-address. " "Consider adding ``--dashboard-address :%d`` " % bokeh_port) dashboard_address = bokeh_port sec = Security(tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key) if not host and (tls_ca_file or tls_cert or tls_key): host = "tls://" if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix="scheduler-") local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info("-" * 47) services = {} if _bokeh: try: from distributed.bokeh.scheduler import BokehScheduler services[("bokeh", dashboard_address)] = ( BokehScheduler, { "prefix": bokeh_prefix }, ) except ImportError as error: if str(error).startswith("No module named"): logger.info( "Web dashboard not loaded. Unable to import bokeh") else: logger.info("Unable to import bokeh: %s" % str(error)) scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file, security=sec) scheduler.start(addr) if not preload: preload = dask.config.get("distributed.scheduler.preload") if not preload_argv: preload_argv = dask.config.get("distributed.scheduler.preload-argv") preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv) logger.info("Local Directory: %26s", local_directory) logger.info("-" * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def main( host, port, bokeh_port, show, dashboard, bokeh, dashboard_prefix, use_xheaders, pid_file, local_directory, tls_ca_file, tls_cert, tls_key, dashboard_address, **kwargs ): g0, g1, g2 = gc.get_threshold() # https://github.com/dask/distributed/issues/1653 gc.set_threshold(g0 * 3, g1 * 3, g2 * 3) enable_proctitle_on_current() enable_proctitle_on_children() if bokeh_port is not None: warnings.warn( "The --bokeh-port flag has been renamed to --dashboard-address. " "Consider adding ``--dashboard-address :%d`` " % bokeh_port ) dashboard_address = bokeh_port if bokeh is not None: warnings.warn( "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. " ) dashboard = bokeh if port is None and (not host or not re.search(r":\d", host)): port = 8786 sec = Security( **{ k: v for k, v in [ ("tls_ca_file", tls_ca_file), ("tls_scheduler_cert", tls_cert), ("tls_scheduler_key", tls_key), ] if v is not None } ) if not host and (tls_ca_file or tls_cert or tls_key): host = "tls://" if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix="scheduler-") local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) loop = IOLoop.current() logger.info("-" * 47) scheduler = Scheduler( loop=loop, security=sec, host=host, port=port, dashboard_address=dashboard_address if dashboard else None, service_kwargs={"dashboard": {"prefix": dashboard_prefix}}, **kwargs, ) logger.info("Local Directory: %26s", local_directory) logger.info("-" * 47) install_signal_handlers(loop) async def run(): await scheduler await scheduler.finished() try: loop.run_sync(run) finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", scheduler.address)
def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key): enable_proctitle_on_current() enable_proctitle_on_children() sec = Security( tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key, ) if not host and (tls_ca_file or tls_cert or tls_key): host = 'tls://' if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix='scheduler-') local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {} if _bokeh: with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_port)] = (BokehScheduler, { 'prefix': bokeh_prefix }) scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file, security=sec) scheduler.start(addr) if not preload: preload = dask.config.get('distributed.scheduler.preload') if not preload_argv: preload_argv = dask.config.get('distributed.scheduler.preload-argv') preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv) logger.info('Local Directory: %26s', local_directory) logger.info('-' * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def main(host, port, bokeh_port, show, dashboard, bokeh, dashboard_prefix, use_xheaders, pid_file, tls_ca_file, tls_cert, tls_key, dashboard_address, **kwargs): g0, g1, g2 = gc.get_threshold( ) # https://github.com/dask/distributed/issues/1653 gc.set_threshold(g0 * 3, g1 * 3, g2 * 3) enable_proctitle_on_current() enable_proctitle_on_children() if bokeh_port is not None: warnings.warn( "The --bokeh-port flag has been renamed to --dashboard-address. " "Consider adding ``--dashboard-address :%d`` " % bokeh_port) dashboard_address = bokeh_port if bokeh is not None: warnings.warn( "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. " ) dashboard = bokeh if port is None and (not host or not re.search(r":\d", host)): port = 8786 sec = { k: v for k, v in [ ("tls_ca_file", tls_ca_file), ("tls_scheduler_cert", tls_cert), ("tls_scheduler_key", tls_key), ] if v is not None } if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ: config = deserialize_for_cli( os.environ["DASK_INTERNAL_INHERIT_CONFIG"]) # Update the global config given priority to the existing global config dask.config.update(dask.config.global_config, config, priority="old") if not host and (tls_ca_file or tls_cert or tls_key): host = "tls://" if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) loop = IOLoop.current() logger.info("-" * 47) scheduler = Scheduler(loop=loop, security=sec, host=host, port=port, dashboard=dashboard, dashboard_address=dashboard_address, http_prefix=dashboard_prefix, **kwargs) logger.info("-" * 47) install_signal_handlers(loop) async def run(): await scheduler await scheduler.finished() try: loop.run_sync(run) finally: scheduler.stop() logger.info("End scheduler at %r", scheduler.address)
def main( host, port, bokeh_port, show, dashboard, dashboard_prefix, use_xheaders, pid_file, scheduler_file, interface, protocol, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key, dashboard_address, ): g0, g1, g2 = gc.get_threshold( ) # https://github.com/dask/distributed/issues/1653 gc.set_threshold(g0 * 3, g1 * 3, g2 * 3) enable_proctitle_on_current() enable_proctitle_on_children() if bokeh_port is not None: warnings.warn( "The --bokeh-port flag has been renamed to --dashboard-address. " "Consider adding ``--dashboard-address :%d`` " % bokeh_port) dashboard_address = bokeh_port if port is None and (not host or not re.search(r":\d", host)): port = 8786 sec = Security(tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key) if not host and (tls_ca_file or tls_cert or tls_key): host = "tls://" if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix="scheduler-") local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) loop = IOLoop.current() logger.info("-" * 47) scheduler = Scheduler( loop=loop, scheduler_file=scheduler_file, security=sec, host=host, port=port, interface=interface, protocol=protocol, dashboard_address=dashboard_address if dashboard else None, service_kwargs={"dashboard": { "prefix": dashboard_prefix }}, ) scheduler.start() if not preload: preload = dask.config.get("distributed.scheduler.preload") if not preload_argv: preload_argv = dask.config.get("distributed.scheduler.preload-argv") preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv) logger.info("Local Directory: %26s", local_directory) logger.info("-" * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", scheduler.address)
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist): given_host = host host = host or get_ip() ip = socket.gethostbyname(host) loop = IOLoop.current() scheduler = Scheduler(center, ip=ip, services={('http', http_port): HTTPScheduler}) scheduler.start(port) if _bokeh: try: import bokeh import distributed.bokeh hosts = [ '%s:%d' % (h, bokeh_port) for h in ['localhost', '127.0.0.1', ip, socket.gethostname(), host] + list(bokeh_whitelist) ] dirname = os.path.dirname(distributed.__file__) paths = [ os.path.join(dirname, 'bokeh', name) for name in ['status', 'tasks'] ] binname = 'bokeh.bat' if 'win' in sys.platform else 'bokeh' binname = os.path.join(os.path.dirname(sys.argv[0]), binname) args = ([binname, 'serve'] + paths + [ '--log-level', 'warning', '--check-unused-sessions=50', '--unused-session-lifetime=1', '--port', str(bokeh_port) ] + sum([['--host', host] for host in hosts], [])) if show: args.append('--show') bokeh_options = { 'host': host if given_host else '127.0.0.1', 'http-port': http_port, 'tcp-port': port, 'bokeh-port': bokeh_port } with open('.dask-web-ui.json', 'w') as f: json.dump(bokeh_options, f, indent=2) if sys.version_info[0] >= 3: from bokeh.command.bootstrap import main ctx = multiprocessing.get_context('spawn') bokeh_proc = ctx.Process(target=main, args=(args, )) bokeh_proc.daemon = True bokeh_proc.start() else: bokeh_proc = subprocess.Popen(args) logger.info(" Bokeh UI at: http://%s:%d/status/" % (ip, bokeh_port)) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) loop.start() loop.close() scheduler.stop() bokeh_proc.terminate() logger.info("End scheduler at %s:%d", ip, port)
def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key): enable_proctitle_on_current() enable_proctitle_on_children() sec = Security(tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key, ) if not host and (tls_ca_file or tls_cert or tls_key): host = 'tls://' if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix='scheduler-') local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {} if _bokeh: try: from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_port)] = (BokehScheduler, {'prefix': bokeh_prefix}) except ImportError as error: if str(error).startswith('No module named'): logger.info('Web dashboard not loaded. Unable to import bokeh') else: logger.info('Unable to import bokeh: %s' % str(error)) scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file, security=sec) scheduler.start(addr) if not preload: preload = dask.config.get('distributed.scheduler.preload') if not preload_argv: preload_argv = dask.config.get('distributed.scheduler.preload-argv') preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv) logger.info('Local Directory: %26s', local_directory) logger.info('-' * 47) install_signal_handlers(loop) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def main(host, port, http_port, bokeh_port, bokeh_internal_port, show, _bokeh, bokeh_whitelist, bokeh_prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload, prefix, tls_ca_file, tls_cert, tls_key): if bokeh_internal_port: print("The --bokeh-internal-port keyword has been removed.\n" "The internal bokeh server is now the default bokeh server.\n" "Use --bokeh-port %d instead" % bokeh_internal_port) sys.exit(1) if prefix: print("The --prefix keyword has moved to --bokeh-prefix") sys.exit(1) sec = Security( tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key, ) if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix='scheduler-') local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {('http', http_port): HTTPScheduler} if _bokeh: with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_port)] = partial(BokehScheduler, prefix=bokeh_prefix) scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file, security=sec) scheduler.start(addr) preload_modules(preload, parameter=scheduler, file_dir=local_directory) logger.info('Local Directory: %26s', local_directory) logger.info('-' * 47) try: loop.start() loop.close() finally: scheduler.stop() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def main(host, port, http_port, bokeh_port, bokeh_external_port, bokeh_internal_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file, scheduler_file, interface, local_directory, preload): if bokeh_internal_port: print("The --bokeh-internal-port keyword has been removed.\n" "The internal bokeh server is now the default bokeh server.\n" "Use --bokeh-port %d instead" % bokeh_internal_port) sys.exit(1) if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) local_directory_created = False if local_directory: if not os.path.exists(local_directory): os.mkdir(local_directory) local_directory_created = True else: local_directory = tempfile.mkdtemp(prefix='scheduler-') local_directory_created = True if local_directory not in sys.path: sys.path.insert(0, local_directory) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {('http', http_port): HTTPScheduler} if _bokeh: with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_port)] = BokehScheduler scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file) scheduler.start(addr) preload_modules(preload, parameter=scheduler, file_dir=local_directory) bokeh_proc = None if _bokeh and bokeh_external_port is not None: if bokeh_external_port == 0: # This is a hack and not robust bokeh_port = open_port() # This port may be taken by the OS try: # before we successfully pass it to Bokeh from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(http_port=http_port, scheduler_address=scheduler.address, bokeh_port=bokeh_external_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info('Local Directory: %26s', local_directory) logger.info('-' * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() if local_directory_created: shutil.rmtree(local_directory) logger.info("End scheduler at %r", addr)
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file): if pid_file: with open(pid_file, "w") as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith("linux"): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) given_host = host host = host or get_ip() if ":" in host and port == 8786: host, port = host.rsplit(":", 1) port = int(port) ip = socket.gethostbyname(host) loop = IOLoop.current() logger.info("-" * 47) services = {("http", http_port): HTTPScheduler} if _bokeh: from distributed.bokeh.scheduler import BokehScheduler services[("bokeh", 8788)] = BokehScheduler scheduler = Scheduler(ip=ip, loop=loop, services=services) scheduler.start(port) bokeh_proc = None if _bokeh: try: from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface( host=host, http_port=http_port, tcp_port=port, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False, ) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info("-" * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %s:%d", ip, port)
def main(host, port, http_port, bokeh_port, bokeh_internal_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file, scheduler_file, interface): if pid_file: with open(pid_file, 'w') as f: f.write(str(os.getpid())) def del_pid_file(): if os.path.exists(pid_file): os.remove(pid_file) atexit.register(del_pid_file) if sys.platform.startswith('linux'): import resource # module fails importing on Windows soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) limit = max(soft, hard // 2) resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) if interface: if host: raise ValueError("Can not specify both interface and host") else: host = get_ip_interface(interface) addr = uri_from_host_port(host, port, 8786) loop = IOLoop.current() logger.info('-' * 47) services = {('http', http_port): HTTPScheduler} if _bokeh: with ignoring(ImportError): from distributed.bokeh.scheduler import BokehScheduler services[('bokeh', bokeh_internal_port)] = BokehScheduler scheduler = Scheduler(loop=loop, services=services, scheduler_file=scheduler_file) scheduler.start(addr) bokeh_proc = None if _bokeh: if bokeh_port == 0: # This is a hack and not robust bokeh_port = open_port() # This port may be taken by the OS try: # before we successfully pass it to Bokeh from distributed.bokeh.application import BokehWebInterface bokeh_proc = BokehWebInterface(http_port=http_port, scheduler_address=scheduler.address, bokeh_port=bokeh_port, bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix, use_xheaders=use_xheaders, quiet=False) except ImportError: logger.info("Please install Bokeh to get Web UI") except Exception as e: logger.warn("Could not start Bokeh web UI", exc_info=True) logger.info('-' * 47) try: loop.start() loop.close() finally: scheduler.stop() if bokeh_proc: bokeh_proc.close() logger.info("End scheduler at %r", addr)