def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
given_host = host
host = host or get_ip()
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center, ip=ip,
services={('http', http_port): HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
scheduler.start(port)
if _bokeh:
try:
import bokeh
import distributed.bokeh
hosts = ['%s:%d' % (h, bokeh_port) for h in
['localhost', '127.0.0.1', ip, socket.gethostname(),
host] + list(bokeh_whitelist)]
dirname = os.path.dirname(distributed.__file__)
paths = [os.path.join(dirname, 'bokeh', name)
for name in ['status', 'tasks']]
binname = sys.argv[0][:-len('dscheduler')] + 'bokeh'
args = ([binname, 'serve'] + paths +
['--log-level', 'warning',
'--check-unused-sessions=50',
'--unused-session-lifetime=1',
'--port', str(bokeh_port)] +
sum([['--host', host] for host in hosts], []))
if show:
args.append('--show')
bokeh_options = {'host': host if given_host else '127.0.0.1',
'http-port': http_port,
'tcp-port': port,
'bokeh-port': bokeh_port}
with open('.dask-web-ui.json', 'w') as f:
json.dump(bokeh_options, f, indent=2)
if sys.version_info[0] >= 3:
from bokeh.command.bootstrap import main
ctx = multiprocessing.get_context('spawn')
bokeh_proc = ctx.Process(target=main, args=(args,))
bokeh_proc.daemon = True
bokeh_proc.start()
else:
bokeh_proc = subprocess.Popen(args)
logger.info(" Bokeh UI at: http://%s:%d/status/"
% (ip, bokeh_port))
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
bokeh_proc.terminate()
logger.info("End scheduler at %s:%d", ip, port)
def main(center, host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist):
given_host = host
host = host or get_ip()
if ':' in host and port == 8786:
host, port = host.rsplit(':', 1)
port = int(port)
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center, ip=ip,
services={('http', http_port): HTTPScheduler},
loop=loop)
scheduler.start(port)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
tcp_port=port, bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist, show=show)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %s:%d", ip, port)
class LocalScheduler():
def __init__(self, dloop):
"""
Create a local scheduler and worker to run commands
"""
self.dloop = dloop
self.loop = dloop.loop
msg.logMessage("Starting scheduler", msg.INFO)
msg.logMessage(self.dloop, msg.DEBUG)
msg.logMessage(self.dloop.loop, msg.DEBUG)
self.s = Scheduler("localhost", loop=self.dloop.loop)
self.s.start(0) # random port
msg.logMessage("End scheduler", msg.INFO)
msg.logMessage("Starting worker", msg.INFO)
self.w = Worker(self.s.ip, self.s.port, ncores=1, loop=self.loop)
self.w.start(0)
self.addr = self.s.ip
self.port = self.s.port
def execute(self):
msg.logMessage("Starting executor", msg.INFO)
self.executor = Executor("{0}:{1}".format(self.addr, self.port))
msg.logMessage("executor..", msg.DEBUG)
def close(self):
self.s.close()
def main(center, host, port, http_port, bokeh_port):
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center, ip=ip,
services={('http', http_port): HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
scheduler.start(port)
try:
import bokeh
except ImportError:
pass
else:
import distributed.diagnostics.bokeh
hosts = ['%s:%d' % (h, bokeh_port) for h in
['localhost', '127.0.0.1', ip, socket.gethostname()]]
dirname = os.path.dirname(distributed.__file__)
path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status')
proc = subprocess.Popen(['bokeh', 'serve', path,
'--log-level', 'warning',
'--port', str(bokeh_port)] +
sum([['--host', host] for host in hosts], []))
distributed.diagnostics.bokeh.server_process = proc # monkey patch
logger.info(" Start Bokeh UI at: http://%s:%d/status/"
% (ip, bokeh_port))
loop.start()
loop.close()
scheduler.stop()
logger.info("End scheduler at %s:%d", ip, port)
def create_scheduler(loop,
scheduler_file=None,
host=None,
bokeh=True,
bokeh_port=8787,
bokeh_prefix=None,
scheduler_port=None):
try:
from distributed.bokeh.scheduler import BokehScheduler
except ImportError:
BokehScheduler = None
if bokeh and BokehScheduler:
services = {
('bokeh', bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
}
else:
services = {}
scheduler = Scheduler(loop=loop,
services=services,
scheduler_file=scheduler_file)
addr = uri_from_host_port(host, scheduler_port, 8786)
scheduler.start(addr)
return scheduler
def _start_scheduler():
logger.info("Starting scheduler...")
loop = IOLoop.current()
s = Scheduler(loop=loop)
s.start("tcp://:6000") # Listen on TCP port 6000
logger.info("Scheduler started")
return s
def main(host, port, http_port, bokeh_port, bokeh_internal_port, show, _bokeh,
bokeh_whitelist, prefix, use_xheaders, pid_file):
if pid_file:
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
def del_pid_file():
if os.path.exists(pid_file):
os.remove(pid_file)
atexit.register(del_pid_file)
if sys.platform.startswith('linux'):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
addr = uri_from_host_port(host, port, 8786)
loop = IOLoop.current()
logger.info('-' * 47)
services = {('http', http_port): HTTPScheduler}
if _bokeh:
with ignoring(ImportError):
from distributed.bokeh.scheduler import BokehScheduler
services[('bokeh', bokeh_internal_port)] = BokehScheduler
scheduler = Scheduler(loop=loop, services=services)
scheduler.start(addr)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(http_port=http_port,
tcp_port=scheduler.port,
bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist,
show=show,
prefix=prefix,
use_xheaders=use_xheaders,
quiet=False)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
logger.info('-' * 47)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %r", addr)
def _start_scheduler(port=6000):
logger = logging.getLogger(__name__)
logger.info("Starting scheduler...")
loop = IOLoop.current()
s = Scheduler(loop=loop)
s.start("tcp://:{}".format(port)) # Listen on port
logger.info("Scheduler started on port {}".format(port))
return s
def test_services():
s = Scheduler(services={'http': HTTPScheduler})
s.start()
try:
assert isinstance(s.services['http'], HTTPServer)
assert s.services['http'].port > 0
finally:
s.close()
def test_services_with_port():
s = Scheduler(services={('http', 9999): HTTPScheduler})
s.start()
try:
assert isinstance(s.services['http'], HTTPServer)
assert s.services['http'].port == 9999
finally:
s.close()
def test_scheduler_file():
with tmpfile() as fn:
s = Scheduler(scheduler_file=fn)
s.start(8008)
w = yield Nanny(scheduler_file=fn)
assert set(s.workers) == {w.worker_address}
yield w.close()
s.stop()
def test_scheduler_file():
with tmpfile() as fn:
s = Scheduler(scheduler_file=fn)
s.start(8008)
w = Nanny(scheduler_file=fn)
yield w._start()
assert s.workers == {w.worker_address}
yield w._close()
s.stop()
def test_scheduler_file():
with tmpfile() as fn:
s = Scheduler(scheduler_file=fn)
s.start(8008)
w = Nanny(scheduler_file=fn)
yield w._start()
assert set(s.workers) == {w.worker_address}
yield w._close()
s.stop()
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist,
prefix, use_xheaders, pid_file):
if pid_file:
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
def del_pid_file():
if os.path.exists(pid_file):
os.remove(pid_file)
atexit.register(del_pid_file)
if sys.platform.startswith('linux'):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
given_host = host
host = host or get_ip()
if ':' in host and port == 8786:
host, port = host.rsplit(':', 1)
port = int(port)
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(ip=ip,
loop=loop,
services={('http', http_port): HTTPScheduler})
scheduler.start(port)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(host=host,
http_port=http_port,
tcp_port=port,
bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist,
show=show,
prefix=prefix,
use_xheaders=use_xheaders,
quiet=False)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %s:%d", ip, port)
def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port):
if interface:
host = get_ip_interface(interface)
else:
host = None
if rank == 0 and scheduler:
try:
from distributed.bokeh.scheduler import BokehScheduler
except ImportError:
services = {}
else:
services = {
('bokeh', bokeh_port): partial(BokehScheduler,
prefix=bokeh_prefix)
}
scheduler = Scheduler(scheduler_file=scheduler_file,
loop=loop,
services=services)
addr = uri_from_host_port(host, None, 8786)
scheduler.start(addr)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
else:
W = Nanny if nanny else Worker
worker = W(scheduler_file=scheduler_file,
loop=loop,
name=rank if scheduler else None,
ncores=nthreads,
local_dir=local_directory,
services={('bokeh', bokeh_worker_port): BokehWorker},
memory_limit=memory_limit)
addr = uri_from_host_port(host, None, 0)
@gen.coroutine
def run():
yield worker._start(addr)
while worker.status != 'closed':
yield gen.sleep(0.2)
try:
loop.run_sync(run)
loop.close()
finally:
pass
@gen.coroutine
def close():
yield worker._close(timeout=2)
loop.run_sync(close)
def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port):
if interface:
host = get_ip_interface(interface)
else:
host = None
if rank == 0 and scheduler:
try:
from distributed.bokeh.scheduler import BokehScheduler
except ImportError:
services = {}
else:
services = {('bokeh', bokeh_port): partial(BokehScheduler,
prefix=bokeh_prefix)}
scheduler = Scheduler(scheduler_file=scheduler_file,
loop=loop,
services=services)
addr = uri_from_host_port(host, None, 8786)
scheduler.start(addr)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
else:
W = Nanny if nanny else Worker
worker = W(scheduler_file=scheduler_file,
loop=loop,
name=rank if scheduler else None,
ncores=nthreads,
local_dir=local_directory,
services={('bokeh', bokeh_worker_port): BokehWorker},
memory_limit=memory_limit)
addr = uri_from_host_port(host, None, 0)
@gen.coroutine
def run():
yield worker._start(addr)
while worker.status != 'closed':
yield gen.sleep(0.2)
try:
loop.run_sync(run)
loop.close()
finally:
pass
@gen.coroutine
def close():
yield worker._close(timeout=2)
loop.run_sync(close)
def main(center, host, port):
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center, services={'http': HTTPScheduler}, ip=ip)
if center:
loop.run_sync(scheduler.sync_center)
scheduler.start(port)
loop.start()
loop.close()
scheduler.stop()
logging.info("End scheduler at %s:%d", ip, port)
def start_scheduler(self, **dask_kw):
"""
Start the scheduler. `dask_kw` are passed straight to the scheduler
constructor.
"""
self._loop = IOLoop.current()
t = Thread(target=self._loop.start)
t.start()
s = Scheduler(loop=self._loop, **dask_kw)
s.start(self.port)
self._scheduler = s
def main(center, host, port, http_port, bokeh_port, show, _bokeh):
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center,
ip=ip,
services={('http', http_port): HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
scheduler.start(port)
if _bokeh:
try:
import bokeh
import distributed.bokeh
hosts = [
'%s:%d' % (h, bokeh_port) for h in
['localhost', '127.0.0.1', ip,
socket.gethostname(), host]
]
dirname = os.path.dirname(distributed.__file__)
paths = [
os.path.join(dirname, 'bokeh', name)
for name in ['status', 'tasks']
]
args = (['bokeh', 'serve'] + paths + [
'--log-level', 'warning', '--check-unused-sessions=50',
'--unused-session-lifetime=1', '--port',
str(bokeh_port)
] + sum([['--host', host] for host in hosts], []))
if show:
args.append('--show')
from bokeh.command.bootstrap import main
proc = multiprocessing.Process(target=main, args=(args, ))
proc.daemon = True
proc.start()
logger.info(" Start Bokeh UI at: http://%s:%d/status/" %
(ip, bokeh_port))
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
proc.terminate()
logger.info("End scheduler at %s:%d", ip, port)
def test_simple(loop):
thread = Thread(target=loop.start)
thread.daemon = True
thread.start()
s = Scheduler(loop=loop)
s.start(0)
with MarathonCluster(s, cpus=1, mem=1000) as mc:
mc.scale_up(1)
while len(s.workers) < 1:
sleep(0.01)
mc.scale_down(s.workers)
while s.workers:
sleep(0.01)
def scheduler(): # pragma: nocover
app_client = skein.ApplicationClient.from_current()
enable_proctitle_on_current()
enable_proctitle_on_children()
if sys.platform.startswith('linux'):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
addr = 'tcp://'
loop = IOLoop.current()
services = {}
bokeh = False
with ignoring(ImportError):
try:
from distributed.dashboard.scheduler import BokehScheduler
except ImportError:
# Old import location
from distributed.bokeh.scheduler import BokehScheduler
services[('bokeh', 0)] = (BokehScheduler, {})
bokeh = True
scheduler = Scheduler(loop=loop, services=services)
scheduler.start(addr)
install_signal_handlers(loop)
# Set dask.dashboard before dask.scheduler since the YarnCluster object
# waits on dask.scheduler only
if bokeh:
bokeh_port = scheduler.services['bokeh'].port
bokeh_host = urlparse(scheduler.address).hostname
bokeh_address = 'http://%s:%d' % (bokeh_host, bokeh_port)
app_client.kv['dask.dashboard'] = bokeh_address.encode()
app_client.kv['dask.scheduler'] = scheduler.address.encode()
try:
loop.start()
loop.close()
finally:
scheduler.stop()
def g():
s = Scheduler(ip='127.0.0.1')
done = s.start()
s.listen(0)
a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1')
yield a._start()
b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip)
yield b._start()
start = time()
try:
while len(s.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(s, a, b)
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
yield s.close()
def start_cluster(ncores, scheduler_addr, loop, security=None,
Worker=Worker, scheduler_kwargs={}, worker_kwargs={}):
s = Scheduler(loop=loop, validate=True, security=security,
**scheduler_kwargs)
done = s.start(scheduler_addr)
workers = [Worker(s.address, ncores=ncore[1], name=i, security=security,
loop=loop, validate=True,
**(merge(worker_kwargs, ncore[2])
if len(ncore) > 2
else worker_kwargs))
for i, ncore in enumerate(ncores)]
for w in workers:
w.rpc = workers[0].rpc
yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]
start = time()
while (len(s.workers) < len(ncores) or
any(comm.comm is None for comm in s.stream_comms.values())):
yield gen.sleep(0.01)
if time() - start > 5:
yield [w._close(timeout=1) for w in workers]
yield s.close(fast=True)
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
def start_cluster(ncores,
scheduler_addr,
loop,
security=None,
Worker=Worker,
scheduler_kwargs={},
worker_kwargs={}):
s = Scheduler(loop=loop,
validate=True,
security=security,
**scheduler_kwargs)
done = s.start(scheduler_addr)
workers = [
Worker(s.address,
ncores=ncore[1],
name=i,
security=security,
loop=loop,
validate=True,
**(merge(worker_kwargs, ncore[2])
if len(ncore) > 2 else worker_kwargs))
for i, ncore in enumerate(ncores)
]
for w in workers:
w.rpc = workers[0].rpc
yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]
start = time()
while len(s.ncores) < len(ncores):
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
def start_cluster(ncores,
loop,
Worker=Worker,
scheduler_kwargs={},
worker_kwargs={}):
s = Scheduler(ip='127.0.0.1', loop=loop, validate=True, **scheduler_kwargs)
done = s.start(0)
workers = [
Worker(s.ip,
s.port,
ncores=ncore[1],
ip=ncore[0],
name=i,
loop=loop,
validate=True,
**(merge(worker_kwargs, ncore[2])
if len(ncore) > 2 else worker_kwargs))
for i, ncore in enumerate(ncores)
]
for w in workers:
w.rpc = workers[0].rpc
yield [w._start() for w in workers]
start = time()
while len(s.ncores) < len(ncores):
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
def g():
s = Scheduler(ip='127.0.0.1')
done = s.start()
s.listen(0)
a = Worker('127.0.0.1', s.port, ncores=2, ip='127.0.0.1')
yield a._start()
b = Worker('127.0.0.1', s.port, ncores=1, ip=b_ip)
yield b._start()
start = time()
try:
while len(s.ncores) < 2:
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
yield f(s, a, b)
finally:
logger.debug("Closing out test cluster")
for w in [a, b]:
with ignoring(TimeoutError, StreamClosedError, OSError):
yield w._close()
if os.path.exists(w.local_dir):
shutil.rmtree(w.local_dir)
yield s.close()
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
s.listen(0)
yield s.sync_center()
done = s.start()
s.update_graph(dsk={'x': (div, 1, 0)}, keys=['x'])
progress = TextProgressBar(['x'],
scheduler=(s.ip, s.port),
start=False,
interval=0.01)
yield progress.listen()
assert progress.status == 'error'
assert progress.stream.closed()
progress = TextProgressBar(['x'],
scheduler=(s.ip, s.port),
start=False,
interval=0.01)
yield progress.listen()
assert progress.status == 'error'
assert progress.stream.closed()
s.close()
yield done
def main(host, port, http_port, bokeh_port, show, _bokeh,
bokeh_whitelist, prefix, use_xheaders, pid_file):
if pid_file:
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
def del_pid_file():
if os.path.exists(pid_file):
os.remove(pid_file)
atexit.register(del_pid_file)
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
given_host = host
host = host or get_ip()
if ':' in host and port == 8786:
host, port = host.rsplit(':', 1)
port = int(port)
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(ip=ip, loop=loop,
services={('http', http_port): HTTPScheduler})
scheduler.start(port)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(host=host, http_port=http_port,
tcp_port=port, bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist, show=show, prefix=prefix,
use_xheaders=use_xheaders, quiet=False)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %s:%d", ip, port)
def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
scheduler):
if interface:
host = get_ip_interface(interface)
else:
host = None
if rank == 0 and scheduler:
scheduler = Scheduler(scheduler_file=scheduler_file,
loop=loop,
services={('bokeh', 8787): BokehScheduler})
addr = uri_from_host_port(host, None, 8786)
scheduler.start(addr)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
else:
worker = Worker(scheduler_file=scheduler_file,
loop=loop,
name=rank if scheduler else None,
ncores=nthreads,
local_dir=local_directory,
services={'bokeh': BokehWorker},
memory_limit=memory_limit)
addr = uri_from_host_port(host, None, 0)
@gen.coroutine
def run():
yield worker._start(addr)
while worker.status != 'closed':
yield gen.sleep(0.2)
try:
loop.run_sync(run)
loop.close()
finally:
pass
@gen.coroutine
def close():
yield worker._close(timeout=2)
loop.run_sync(close)
def main(center, host, port, http_port, bokeh_port, show, _bokeh):
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center, ip=ip,
services={('http', http_port): HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
scheduler.start(port)
if _bokeh:
try:
import bokeh
import distributed.bokeh
hosts = ['%s:%d' % (h, bokeh_port) for h in
['localhost', '127.0.0.1', ip, socket.gethostname(), host]]
dirname = os.path.dirname(distributed.__file__)
paths = [os.path.join(dirname, 'bokeh', name)
for name in ['status', 'tasks']]
args = (['bokeh', 'serve'] + paths +
['--log-level', 'warning',
'--check-unused-sessions=50',
'--unused-session-lifetime=1',
'--port', str(bokeh_port)] +
sum([['--host', host] for host in hosts], []))
if show:
args.append('--show')
from bokeh.command.bootstrap import main
proc = multiprocessing.Process(target=main, args=(args,))
proc.daemon = True
proc.start()
logger.info(" Start Bokeh UI at: http://%s:%d/status/"
% (ip, bokeh_port))
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
proc.terminate()
logger.info("End scheduler at %s:%d", ip, port)
def main():
app_client = skein.ApplicationClient.from_current()
enable_proctitle_on_current()
enable_proctitle_on_children()
if sys.platform.startswith('linux'):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
addr = uri_from_host_port('', None, 0)
loop = IOLoop.current()
services = {}
bokeh = False
with ignoring(ImportError):
from distributed.bokeh.scheduler import BokehScheduler
services[('bokeh', 0)] = (BokehScheduler, {})
bokeh = True
scheduler = Scheduler(loop=loop, services=services)
scheduler.start(addr)
install_signal_handlers(loop)
app_client.kv['dask.scheduler'] = scheduler.address.encode()
if bokeh:
bokeh_port = scheduler.services['bokeh'].port
bokeh_host = urlparse(scheduler.address).hostname
bokeh_address = 'http://%s:%d' % (bokeh_host, bokeh_port)
app_client.kv['dask.dashboard'] = bokeh_address.encode()
try:
loop.start()
loop.close()
finally:
scheduler.stop()
def main(center, port):
loop = IOLoop.current()
scheduler = Scheduler(center, services={'http': HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
done = scheduler.start(port)
loop.start()
loop.close()
scheduler.stop()
logging.info("End scheduler at %s:%d", ip, port)
def test_adapt(loop):
thread = Thread(target=loop.start)
thread.daemon = True
thread.start()
s = Scheduler(loop=loop)
s.start(0)
with MarathonCluster(s, cpus=1, mem=1000) as mc:
ac = Adaptive(s, mc)
with Client(s.address, loop=loop) as c:
assert not s.ncores
x = c.submit(lambda x: x + 1, 1)
x.result()
assert len(s.ncores) == 1
del x
start = time()
while s.ncores:
sleep(0.01)
assert time() < start + 5
def main(center, port):
loop = IOLoop.current()
scheduler = Scheduler(center, services={'http': HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
done = scheduler.start(port)
loop.start()
loop.close()
scheduler.stop()
logging.info("End scheduler at %s:%d", ip, port)
def start_dask_scheduler():
"""
Start the Scheduler in this subprocess.
Context: A Dask Client is started in the doctor server and needs
to connect to the Dask Scheduler
:return:
"""
services = {('bokeh', 8787): (BokehScheduler, {'prefix': None})}
scheduler = Scheduler(services=services)
scheduler_location = 'tcp://:%s' % scheduler_port
scheduler.start(scheduler_location)
# Tornado IOLoops are a way for Schedulers, Workers and Clients to communicate.
# Here, we get the IOLoop of the process and start it.
# It is used by the Scheduler to communicate with Workers from the outside.
loop = IOLoop.current()
loop.start()
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist,
prefix, use_xheaders):
given_host = host
host = host or get_ip()
if ':' in host and port == 8786:
host, port = host.rsplit(':', 1)
port = int(port)
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(ip=ip,
loop=loop,
services={('http', http_port): HTTPScheduler})
scheduler.start(port)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(host=host,
http_port=http_port,
tcp_port=port,
bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist,
show=show,
prefix=prefix,
use_xheaders=use_xheaders,
quiet=False)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
loop.start()
loop.close()
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %s:%d", ip, port)
def start_cluster(ncores, Worker=Worker):
s = Scheduler(ip='127.0.0.1')
done = s.start(0)
workers = [Worker(s.ip, s.port, ncores=v, ip=k) for k, v in ncores]
yield [w._start() for w in workers]
start = time()
while len(s.ncores) < len(ncores):
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
async def start_scheduler(gateway, security, exit_on_failure=True):
loop = IOLoop.current()
plugin = GatewaySchedulerPlugin(gateway, loop)
services = {("gateway", 0): (GatewaySchedulerService, {"plugin": plugin})}
bokeh = False
with ignoring(ImportError):
try:
from distributed.dashboard.scheduler import BokehScheduler
except ImportError:
# Old import location
from distributed.bokeh.scheduler import BokehScheduler
services[("bokeh", 0)] = (BokehScheduler, {})
bokeh = True
scheduler = Scheduler(loop=loop, services=services, security=security)
scheduler.add_plugin(plugin)
scheduler.start("tls://")
host = urlparse(scheduler.address).hostname
gateway_port = scheduler.services["gateway"].port
api_address = "http://%s:%d" % (host, gateway_port)
if bokeh:
bokeh_port = scheduler.services["bokeh"].port
bokeh_address = "http://%s:%d" % (host, bokeh_port)
else:
bokeh_address = ""
try:
await gateway.send_addresses(scheduler.address, bokeh_address,
api_address)
except Exception as exc:
logger.error("Failed to send addresses to gateway", exc_info=exc)
if exit_on_failure:
sys.exit(1)
return scheduler
def main(center, host, port, http_port, bokeh_port):
ip = socket.gethostbyname(host)
loop = IOLoop.current()
scheduler = Scheduler(center,
ip=ip,
services={('http', http_port): HTTPScheduler})
if center:
loop.run_sync(scheduler.sync_center)
scheduler.start(port)
try:
import bokeh
except ImportError:
pass
else:
import distributed.diagnostics.bokeh
hosts = [
'%s:%d' % (h, bokeh_port)
for h in ['localhost', '127.0.0.1', ip,
socket.gethostname()]
]
dirname = os.path.dirname(distributed.__file__)
path = os.path.join(dirname, 'diagnostics', 'bokeh', 'status')
proc = subprocess.Popen([
'bokeh', 'serve', path, '--log-level', 'warning', '--port',
str(bokeh_port)
] + sum([['--host', host] for host in hosts], []))
distributed.diagnostics.bokeh.server_process = proc # monkey patch
logger.info(" Start Bokeh UI at: http://%s:%d/status/" %
(ip, bokeh_port))
loop.start()
loop.close()
scheduler.stop()
logger.info("End scheduler at %s:%d", ip, port)
def start_cluster(ncores, loop, Worker=Worker):
s = Scheduler(ip='127.0.0.1', loop=loop)
done = s.start(0)
workers = [Worker(s.ip, s.port, ncores=v, ip=k, name=i, loop=loop)
for i, (k, v) in enumerate(ncores)]
yield [w._start() for w in workers]
start = time()
while len(s.ncores) < len(ncores):
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s.sync_center()
done = s.start(0)
progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False,
interval=0.01)
yield progress.listen()
assert progress.status == 'finished'
check_bar_completed(capsys)
s.close()
yield done
def run_scheduler(q, scheduler_port=0, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
scheduler = Scheduler(loop=loop, validate=True, **kwargs)
done = scheduler.start(scheduler_port)
q.put(scheduler.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_scheduler(q, nputs, **kwargs):
from distributed import Scheduler
# On Python 2.7 and Unix, fork() is used to spawn child processes,
# so avoid inheriting the parent's IO loop.
with pristine_loop() as loop:
scheduler = Scheduler(validate=True, **kwargs)
done = scheduler.start('127.0.0.1')
for i in range(nputs):
q.put(scheduler.address)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_scheduler(q, nputs, **kwargs):
from distributed import Scheduler
# On Python 2.7 and Unix, fork() is used to spawn child processes,
# so avoid inheriting the parent's IO loop.
with pristine_loop() as loop:
scheduler = Scheduler(validate=True, **kwargs)
done = scheduler.start('127.0.0.1')
for i in range(nputs):
q.put(scheduler.address)
try:
loop.start()
finally:
loop.close(all_fds=True)
def f():
s = Scheduler(loop=loop)
done = s.start(0)
a = Worker(s.ip, s.port, loop=loop, ncores=1)
b = Worker(s.ip, s.port, loop=loop, ncores=1)
yield [a._start(0), b._start(0)]
progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False,
interval=0.01)
yield progress.listen()
assert progress.status == 'finished'
check_bar_completed(capsys)
yield [a._close(), b._close()]
s.close()
yield done
def run_scheduler(q, scheduler_port=0, center_port=None, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = ('127.0.0.1', center_port) if center_port else None
scheduler = Scheduler(center=center, loop=loop, validate=True, **kwargs)
done = scheduler.start(scheduler_port)
q.put(scheduler.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_scheduler(q, center_port=None, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = ('127.0.0.1', center_port) if center_port else None
scheduler = Scheduler(center=center, **kwargs)
scheduler.listen(0)
if center_port:
loop.run_sync(scheduler.sync_center)
done = scheduler.start(0)
q.put(scheduler.port)
loop.start()
def start_cluster(ncores, loop, Worker=Worker, scheduler_kwargs={},
worker_kwargs={}):
s = Scheduler(ip='127.0.0.1', loop=loop, validate=True, **scheduler_kwargs)
done = s.start(0)
workers = [Worker(s.ip, s.port, ncores=ncore[1], ip=ncore[0], name=i,
loop=loop, validate=True,
**(merge(worker_kwargs, ncore[2])
if len(ncore) > 2
else worker_kwargs))
for i, ncore in enumerate(ncores)]
for w in workers:
w.rpc = workers[0].rpc
yield [w._start() for w in workers]
start = time()
while len(s.ncores) < len(ncores):
yield gen.sleep(0.01)
if time() - start > 5:
raise Exception("Cluster creation timeout")
raise gen.Return((s, workers))
def f(c, a, b):
s = Scheduler((c.ip, c.port), loop=loop)
yield s.sync_center()
done = s.start()
s.update_graph(dsk={'x': (div, 1, 0)},
keys=['x'])
progress = TextProgressBar(['x'], scheduler=(s.ip, s.port),
start=False, interval=0.01)
yield progress.listen()
assert progress.status == 'error'
assert progress.stream.closed()
progress = TextProgressBar(['x'], scheduler=(s.ip, s.port),
start=False, interval=0.01)
yield progress.listen()
assert progress.status == 'error'
assert progress.stream.closed()
s.close()
yield done
def _create_scheduler(self):
scheduler = Scheduler(ip=self.ip)
scheduler.start(self.port)
return scheduler
def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix,
use_xheaders, pid_file, scheduler_file, interface,
local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key):
enable_proctitle_on_current()
enable_proctitle_on_children()
sec = Security(tls_ca_file=tls_ca_file,
tls_scheduler_cert=tls_cert,
tls_scheduler_key=tls_key,
)
if not host and (tls_ca_file or tls_cert or tls_key):
host = 'tls://'
if pid_file:
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
def del_pid_file():
if os.path.exists(pid_file):
os.remove(pid_file)
atexit.register(del_pid_file)
local_directory_created = False
if local_directory:
if not os.path.exists(local_directory):
os.mkdir(local_directory)
local_directory_created = True
else:
local_directory = tempfile.mkdtemp(prefix='scheduler-')
local_directory_created = True
if local_directory not in sys.path:
sys.path.insert(0, local_directory)
if sys.platform.startswith('linux'):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
if interface:
if host:
raise ValueError("Can not specify both interface and host")
else:
host = get_ip_interface(interface)
addr = uri_from_host_port(host, port, 8786)
loop = IOLoop.current()
logger.info('-' * 47)
services = {}
if _bokeh:
try:
from distributed.bokeh.scheduler import BokehScheduler
services[('bokeh', bokeh_port)] = (BokehScheduler,
{'prefix': bokeh_prefix})
except ImportError as error:
if str(error).startswith('No module named'):
logger.info('Web dashboard not loaded. Unable to import bokeh')
else:
logger.info('Unable to import bokeh: %s' % str(error))
scheduler = Scheduler(loop=loop, services=services,
scheduler_file=scheduler_file,
security=sec)
scheduler.start(addr)
if not preload:
preload = dask.config.get('distributed.scheduler.preload')
if not preload_argv:
preload_argv = dask.config.get('distributed.scheduler.preload-argv')
preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv)
logger.info('Local Directory: %26s', local_directory)
logger.info('-' * 47)
install_signal_handlers(loop)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
if local_directory_created:
shutil.rmtree(local_directory)
logger.info("End scheduler at %r", addr)
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist, prefix, use_xheaders, pid_file):
if pid_file:
with open(pid_file, "w") as f:
f.write(str(os.getpid()))
def del_pid_file():
if os.path.exists(pid_file):
os.remove(pid_file)
atexit.register(del_pid_file)
if sys.platform.startswith("linux"):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
given_host = host
host = host or get_ip()
if ":" in host and port == 8786:
host, port = host.rsplit(":", 1)
port = int(port)
ip = socket.gethostbyname(host)
loop = IOLoop.current()
logger.info("-" * 47)
services = {("http", http_port): HTTPScheduler}
if _bokeh:
from distributed.bokeh.scheduler import BokehScheduler
services[("bokeh", 8788)] = BokehScheduler
scheduler = Scheduler(ip=ip, loop=loop, services=services)
scheduler.start(port)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(
host=host,
http_port=http_port,
tcp_port=port,
bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist,
show=show,
prefix=prefix,
use_xheaders=use_xheaders,
quiet=False,
)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
logger.info("-" * 47)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %s:%d", ip, port)
def test_tls_scheduler(security, loop):
s = Scheduler(security=security, loop=loop)
s.start('localhost')
assert s.address.startswith('tls')
s.close()
