class LocalScheduler(): def __init__(self, dloop): """ Create a local scheduler and worker to run commands """ self.dloop = dloop self.loop = dloop.loop msg.logMessage("Starting scheduler", msg.INFO) msg.logMessage(self.dloop, msg.DEBUG) msg.logMessage(self.dloop.loop, msg.DEBUG) self.s = Scheduler("localhost", loop=self.dloop.loop) self.s.start(0) # random port msg.logMessage("End scheduler", msg.INFO) msg.logMessage("Starting worker", msg.INFO) self.w = Worker(self.s.ip, self.s.port, ncores=1, loop=self.loop) self.w.start(0) self.addr = self.s.ip self.port = self.s.port def execute(self): msg.logMessage("Starting executor", msg.INFO) self.executor = Executor("{0}:{1}".format(self.addr, self.port)) msg.logMessage("executor..", msg.DEBUG) def close(self): self.s.close()
def start_worker(address): import distributed from distributed import Worker from tornado.ioloop import IOLoop if getattr(distributed, 'global_worker', False): return ['already occupied'] loop = IOLoop.current() w = Worker(address, loop=loop) w.start(0) print("Started worker") distributed.global_worker = w from tornado import gen @gen.coroutine def _(): while w.status != 'closed': yield gen.sleep(0.1) loop.run_sync(_) distributed.global_worker = False return ['completed']
def get_repl_worker(): loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() w = Worker('tcp://127.0.0.1:8786', loop=loop) w.start() # choose randomly assigned port return w
def get_repl_worker(): loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() w = Worker('tcp://127.0.0.1:8786', loop=loop) w.start() # choose randomly assigned port return w
def run_worker(port, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop import logging logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs) worker.start() IOLoop.current().start() IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
def run_worker(port, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop import logging logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs) worker.start() IOLoop.current().start() IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
def run_worker(port, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs) worker.start() loop.start()
def create_worker(num_worker, server_ip, server_port): """ Instantiate the dask workers Args: num_worker: number of workers server_ip: dask scheduler IP server_port: dask port """ for i in range(int(num_worker)): print "-- worker initializing --" dask_server = Worker('tcp://' + server_ip + ":" + str(server_port), loop=loop) dask_server.start()
# encoding: utf-8 import tornado.web import tornado.ioloop import tornado.gen import json import requests import traceback from cesium_app.config import cfg IP = '127.0.0.1' PORT_SCHEDULER = 63500 if __name__ == "__main__": loop = tornado.ioloop.IOLoop.current() from distributed import Scheduler s = Scheduler(loop=loop) s.start(PORT_SCHEDULER) print('Task scheduler listening on port {}'.format(PORT_SCHEDULER)) from distributed import Worker w = Worker('127.0.0.1', PORT_SCHEDULER, loop=loop, ncores=1) w.start(0) print('Single worker activated') print('Starting main loop...') loop.start()
delay = 5 # seconds while True: sleep(delay) logging.info("Dask Worker status : %s" % worker.status) if worker.status in ('closing', 'closed'): logging.info( "Exiting the Dask Worker via watchdog using SIGKILL") os.kill(pid, signal.SIGKILL) worker_watchdog_thread = Thread(target=worker_watchdog) # We activate the daemon explicitly so that this thread # is stopped when the main process exits. # see: https://docs.python.org/2/library/threading.html#threading.Thread.daemon worker_watchdog_thread.daemon = True logging.info("Launching Dask Worker via Python API") logging.info("Python version:") logging.info(platform.sys.version) logging.info("Dask version: %s" % dask.__version__) logging.info("Distributed version: %s" % distributed.__version__) worker.start() logging.info("Launching Worker Watchdog Thread") worker_watchdog_thread.start() loop = IOLoop.current() loop.start() logging.info("Exiting the Dask Worker normally")
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Fri Feb 22 17:15:29 2019 @author: cliffk """ from distributed import Worker from tornado.ioloop import IOLoop from threading import Thread loop = IOLoop.current() t = Thread(target=loop.start, daemon=True) t.start() w = Worker('tcp://127.0.0.1:8786', loop=loop) w.start() # choose randomly assigned port