class LocalScheduler():
def __init__(self, dloop):
"""
Create a local scheduler and worker to run commands
"""
self.dloop = dloop
self.loop = dloop.loop
msg.logMessage("Starting scheduler", msg.INFO)
msg.logMessage(self.dloop, msg.DEBUG)
msg.logMessage(self.dloop.loop, msg.DEBUG)
self.s = Scheduler("localhost", loop=self.dloop.loop)
self.s.start(0) # random port
msg.logMessage("End scheduler", msg.INFO)
msg.logMessage("Starting worker", msg.INFO)
self.w = Worker(self.s.ip, self.s.port, ncores=1, loop=self.loop)
self.w.start(0)
self.addr = self.s.ip
self.port = self.s.port
def execute(self):
msg.logMessage("Starting executor", msg.INFO)
self.executor = Executor("{0}:{1}".format(self.addr, self.port))
msg.logMessage("executor..", msg.DEBUG)
def close(self):
self.s.close()
def start_worker(address):
import distributed
from distributed import Worker
from tornado.ioloop import IOLoop
if getattr(distributed, 'global_worker', False):
return ['already occupied']
loop = IOLoop.current()
w = Worker(address, loop=loop)
w.start(0)
print("Started worker")
distributed.global_worker = w
from tornado import gen
@gen.coroutine
def _():
while w.status != 'closed':
yield gen.sleep(0.1)
loop.run_sync(_)
distributed.global_worker = False
return ['completed']
def get_repl_worker():
loop = IOLoop.current()
t = Thread(target=loop.start, daemon=True)
t.start()
w = Worker('tcp://127.0.0.1:8786', loop=loop)
w.start() # choose randomly assigned port
return w
def get_repl_worker():
loop = IOLoop.current()
t = Thread(target=loop.start, daemon=True)
t.start()
w = Worker('tcp://127.0.0.1:8786', loop=loop)
w.start() # choose randomly assigned port
return w
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop
import logging
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
worker.start()
IOLoop.current().start()
IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop
import logging
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
worker.start()
IOLoop.current().start()
IOLoop.current().close() # Never reached. TODO: clean shutdown of IOLoop
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', port, '127.0.0.1', center_port, **kwargs)
worker.start()
loop.start()
def create_worker(num_worker, server_ip, server_port):
""" Instantiate the dask workers
Args:
num_worker: number of workers
server_ip: dask scheduler IP
server_port: dask port
"""
for i in range(int(num_worker)):
print "-- worker initializing --"
dask_server = Worker('tcp://' + server_ip + ":" + str(server_port),
loop=loop)
dask_server.start()
# encoding: utf-8
import tornado.web
import tornado.ioloop
import tornado.gen
import json
import requests
import traceback
from cesium_app.config import cfg
IP = '127.0.0.1'
PORT_SCHEDULER = 63500
if __name__ == "__main__":
loop = tornado.ioloop.IOLoop.current()
from distributed import Scheduler
s = Scheduler(loop=loop)
s.start(PORT_SCHEDULER)
print('Task scheduler listening on port {}'.format(PORT_SCHEDULER))
from distributed import Worker
w = Worker('127.0.0.1', PORT_SCHEDULER, loop=loop, ncores=1)
w.start(0)
print('Single worker activated')
print('Starting main loop...')
loop.start()
delay = 5 # seconds
while True:
sleep(delay)
logging.info("Dask Worker status : %s" % worker.status)
if worker.status in ('closing', 'closed'):
logging.info(
"Exiting the Dask Worker via watchdog using SIGKILL")
os.kill(pid, signal.SIGKILL)
worker_watchdog_thread = Thread(target=worker_watchdog)
# We activate the daemon explicitly so that this thread
# is stopped when the main process exits.
# see: https://docs.python.org/2/library/threading.html#threading.Thread.daemon
worker_watchdog_thread.daemon = True
logging.info("Launching Dask Worker via Python API")
logging.info("Python version:")
logging.info(platform.sys.version)
logging.info("Dask version: %s" % dask.__version__)
logging.info("Distributed version: %s" % distributed.__version__)
worker.start()
logging.info("Launching Worker Watchdog Thread")
worker_watchdog_thread.start()
loop = IOLoop.current()
loop.start()
logging.info("Exiting the Dask Worker normally")
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 22 17:15:29 2019
@author: cliffk
"""
from distributed import Worker
from tornado.ioloop import IOLoop
from threading import Thread
loop = IOLoop.current()
t = Thread(target=loop.start, daemon=True)
t.start()
w = Worker('tcp://127.0.0.1:8786', loop=loop)
w.start() # choose randomly assigned port
