async def client(loop:IOLoop): try: print("try to connect {0:s}:{1:s}".format(args.host,args.port)) stream = await clientObj.connect(args.host,args.port) #type:tornado.concurrent.futures.Future stream.set_nodelay(True) async def handle(): while True: command=Q.get().split(' ') act=command[0] if act=='p': print(goodsTable) elif act=='s': good=goodsList[int(command[1])] RFID = bytes.fromhex(good[0]) msg =SSCEncoder.SSCFrameEncoder(RFID,CartID,op,opData) print("sengMessage:",binascii.b2a_hex(msg)) await stream.write(msg) elif act=='d': msg = command[1].encode() await stream.write(msg) def waitKey(): while True: Q.put_nowait(input('>')) loop.add_callback(handle) loop.run_in_executor(None,waitKey) except StreamClosedError : print("error connecting, try in 5 sec") await gen.sleep(5) loop.add_callback(functools.partial(client,loop))
def run(case_data, **server_kwargs): """ Real test function called for a given case data. Load a server (with optional given server kwargs), call function main(case_data) as client code and wait for main function to terminate. :type case_data: CaseData """ print() io_loop = IOLoop() io_loop.make_current() common.Tornado.stop_loop_on_callback_error(io_loop) case_data.io_loop = io_loop case_data.test_server = Server(**server_kwargs) @gen.coroutine def coroutine_func(): """ Concrete call to main function. """ yield main(case_data) case_data.io_loop.stop() print('Finished', case_data.case_name, 'at', common.timestamp_microseconds()) io_loop.add_callback(coroutine_func) case_data.test_server.start(case_data.port, io_loop) case_data.io_loop.clear_current() case_data.io_loop.close() case_data.test_server.backend.http_server.stop()
def connect(self): """ Connect to the server """ assert self._conn is None assert self._conn_ioloop is None future = Future() io_loop = IOLoop() thread = threading.Thread(target=io_loop.start, name='DatabaseThread') thread.start() def _connect(): try: conn = connect(**self._db_args) self._conn = conn self._conn_ioloop = io_loop self._conn_thread = thread future.set_result(None) except Exception as ex: io_loop.stop() future.set_exception(ex) io_loop.add_callback(_connect) yield future
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port, local_dir, services): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(center_ip, center_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start() # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({ 'port': worker.port, 'dir': worker.local_dir }) # pragma: no cover loop.add_callback(start) # pragma: no cover loop.start() # pragma: no cover
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port, worker_port, local_dir, services, name): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(center_ip, center_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover loop.add_callback(start) # pragma: no cover with ignoring(KeyboardInterrupt): loop.start() # pragma: no cover
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port, local_dir): """ Function run by the Nanny when creating the worker """ from distributed import Worker from tornado.ioloop import IOLoop IOLoop.clear_instance() loop = IOLoop() loop.make_current() worker = Worker(center_ip, center_port, ncores=ncores, ip=ip, nanny_port=nanny_port, local_dir=local_dir) @gen.coroutine def start(): try: yield worker._start() except Exception as e: logger.exception(e) q.put(e) else: assert worker.port q.put({'port': worker.port, 'dir': worker.local_dir}) loop.add_callback(start) loop.start()
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port, worker_port, local_dir, services, name, memory_limit): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name, memory_limit=memory_limit, loop=loop) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover loop.add_callback(start) # pragma: no cover try: loop.start() # pragma: no cover finally: loop.stop() loop.close(all_fds=True)
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ == 'TwistedIOLoop': # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop') self.server_ioloop = IOLoop() @gen.coroutine def init_server(): sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app) self.server.add_socket(sock) self.server_ioloop.run_sync(init_server) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() # Delay the shutdown of the IOLoop by several iterations because # the server may still have some cleanup work left when # the client finishes with the response (this is noticeable # with http/2, which leaves a Future with an unexamined # StreamClosedError on the loop). @gen.coroutine def slow_stop(): # The number of iterations is difficult to predict. Typically, # one is sufficient, although sometimes it needs more. for i in range(5): yield self.server_ioloop.stop() self.server_ioloop.add_callback(slow_stop) self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return 'http://127.0.0.1:%d%s' % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url('/')) self.assertEqual(b'Hello world!', response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url('/notfound')) self.assertEqual(assertion.exception.code, 404)
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port, local_dir, services): """ Function run by the Nanny when creating the worker """ from distributed import Worker from tornado.ioloop import IOLoop IOLoop.clear_instance() loop = IOLoop() loop.make_current() worker = Worker(center_ip, center_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services) @gen.coroutine def start(): try: yield worker._start() except Exception as e: logger.exception(e) q.put(e) else: assert worker.port q.put({'port': worker.port, 'dir': worker.local_dir}) loop.add_callback(start) loop.start()
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): self.server_ioloop = IOLoop() event = threading.Event() @gen.coroutine def init_server(): sock, self.port = bind_unused_port() app = Application([("/", HelloWorldHandler)]) self.server = HTTPServer(app) self.server.add_socket(sock) event.set() def start(): self.server_ioloop.run_sync(init_server) self.server_ioloop.start() self.server_thread = threading.Thread(target=start) self.server_thread.start() event.wait() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() # Delay the shutdown of the IOLoop by several iterations because # the server may still have some cleanup work left when # the client finishes with the response (this is noticeable # with http/2, which leaves a Future with an unexamined # StreamClosedError on the loop). @gen.coroutine def slow_stop(): # The number of iterations is difficult to predict. Typically, # one is sufficient, although sometimes it needs more. for i in range(5): yield self.server_ioloop.stop() self.server_ioloop.add_callback(slow_stop) self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return "http://127.0.0.1:%d%s" % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url("/")) self.assertEqual(b"Hello world!", response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url("/notfound")) self.assertEqual(assertion.exception.code, 404)
def wrapper(): ioloop = IOLoop() @coroutine def looped(): yield func() ioloop.stop() ioloop.add_callback(looped) thread = Thread(target=ioloop.start) thread.start()
def _run(): io_loop = IOLoop() io_loop.make_current() io_loop.add_callback(lambda: evt.set()) with mock.patch.dict(os.environ, env): app = self._app = MockSingleUserServer() app.initialize(args) app.start()
def _run(): io_loop = IOLoop() io_loop.make_current() io_loop.add_callback(lambda : evt.set()) with mock.patch.dict(os.environ, env): app = self._app = MockSingleUserServer() app.initialize(args) app.start()
async def request_periodically(port: int, loop: IOLoop): client = AsyncHTTPClient() while True: gets = randint(0, 10) posts = randint(0, 10) for _ in range(gets): loop.add_callback(get, client, port) for _ in range(posts): loop.add_callback(post, client, port) await sleep(random() * 5.5)
def wrapper(): ioloop = IOLoop() @coroutine def looped(): yield func() ioloop.stop() ioloop.add_callback(looped) thread = Thread(target=ioloop.start) thread.start()
class FakeServerContext(object): def __init__(self, monkeypatch, fail_these, expected_basename): self._monkeypatch = monkeypatch self._fail_these = fail_these self._expected_basename = expected_basename self._url = None self._loop = None self._started = threading.Condition() self._thread = threading.Thread(target=self._run) def __exit__(self, type, value, traceback): if self._loop is not None: # we can ONLY use add_callback here, since the loop is # running in a different thread. self._loop.add_callback(self._stop) self._thread.join() def __enter__(self): self._started.acquire() self._thread.start() self._started.wait() self._started.release() _monkeypatch_client_config(self._monkeypatch, self._url) return self._url def _run(self): self._loop = IOLoop() self._server = FakeAnacondaServer( io_loop=self._loop, fail_these=self._fail_these, expected_basename=self._expected_basename) self._url = self._server.url def notify_started(): self._started.acquire() self._started.notify() self._started.release() self._loop.add_callback(notify_started) self._loop.start() # done self._server.unlisten() def _stop(self): def really_stop(): if self._loop is not None: self._loop.stop() self._loop = None # the delay allows pending next-tick things to go ahead # and happen, which may avoid some problems with trying to # output to stdout after pytest closes it if self._loop is not None: self._loop.call_later(delay=0.05, callback=really_stop)
def _run(): asyncio.set_event_loop(asyncio.new_event_loop()) io_loop = IOLoop() io_loop.make_current() io_loop.add_callback(lambda: evt.set()) with mock.patch.dict(os.environ, env): app = self._app = MockSingleUserServer() app.initialize(args) assert app.hub_auth.oauth_client_id assert app.hub_auth.api_token app.start()
def _run(): asyncio.set_event_loop(asyncio.new_event_loop()) io_loop = IOLoop() io_loop.make_current() io_loop.add_callback(lambda: evt.set()) with mock.patch.dict(os.environ, env): app = self._app = MockSingleUserServer() app.initialize(args) assert app.hub_auth.oauth_client_id assert app.hub_auth.api_token app.start()
class FakeServerContext(object): def __init__(self, monkeypatch, fail_these, expected_basename): self._monkeypatch = monkeypatch self._fail_these = fail_these self._expected_basename = expected_basename self._url = None self._loop = None self._started = threading.Condition() self._thread = threading.Thread(target=self._run) def __exit__(self, type, value, traceback): if self._loop is not None: # we can ONLY use add_callback here, since the loop is # running in a different thread. self._loop.add_callback(self._stop) self._thread.join() def __enter__(self): self._started.acquire() self._thread.start() self._started.wait() self._started.release() _monkeypatch_client_config(self._monkeypatch, self._url) return self._url def _run(self): self._loop = IOLoop() self._server = FakeAnacondaServer(io_loop=self._loop, fail_these=self._fail_these, expected_basename=self._expected_basename) self._url = self._server.url def notify_started(): self._started.acquire() self._started.notify() self._started.release() self._loop.add_callback(notify_started) self._loop.start() # done self._server.unlisten() def _stop(self): def really_stop(): if self._loop is not None: self._loop.stop() self._loop = None # the delay allows pending next-tick things to go ahead # and happen, which may avoid some problems with trying to # output to stdout after pytest closes it if self._loop is not None: self._loop.call_later(delay=0.05, callback=really_stop)
class _Executor(Thread): """ Tasks executor. Task is executed in ioloop for easier stopping it. Subprocess based tasks are killed external """ def __init__(self, task, number, *args, **kwargs): super(_Executor, self).__init__(*args, **kwargs) self.ioloop = None self.task = task self.number = number def run(self): if self.task.cancelled: return self.task.set_executor(executor=self) self.ioloop = IOLoop() self.ioloop.make_current() self.ioloop.add_callback(self.execute) self.ioloop.start() self.task.clear() self.ioloop.clear_current() async def execute(self): """ Update task and stop ioloop """ try: await self.task() await self.task.post_run() except subprocess.CalledProcessError as exception: log.warning('%s', exception) except: log.exception("Exception while executing task on worker %s", self.number) finally: self.ioloop.stop() self.task.finish_time = int(time.time()) def stop(self): """ Stop task. Important especially for Subprocess based tasks """ self.task.kill() # As Subprocess based tasks generate traffic only using external tool, they should exit gracefully if not isinstance(self.task, (CommandTask, NmapPortInfoTask, PortScanTask)): self.ioloop.stop() def __str__(self): return str(self.task)
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ in ('TwistedIOLoop', 'AsyncIOMainLoop'): # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. # AsyncIOMainLoop doesn't work with the default policy # (although it could with some tweaks to this test and a # policy that created loops for non-main threads). raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop or ' 'AsyncIOMainLoop') self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() # Delay the shutdown of the IOLoop by one iteration because # the server may still have some cleanup work left when # the client finishes with the response (this is noticable # with http/2, which leaves a Future with an unexamined # StreamClosedError on the loop). self.server_ioloop.add_callback(self.server_ioloop.stop) self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return 'http://127.0.0.1:%d%s' % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url('/')) self.assertEqual(b'Hello world!', response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url('/notfound')) self.assertEqual(assertion.exception.code, 404)
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ in ('TwistedIOLoop', 'AsyncIOMainLoop'): # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. # AsyncIOMainLoop doesn't work with the default policy # (although it could with some tweaks to this test and a # policy that created loops for non-main threads). raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop or ' 'AsyncIOMainLoop') self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() # Delay the shutdown of the IOLoop by one iteration because # the server may still have some cleanup work left when # the client finishes with the response (this is noticable # with http/2, which leaves a Future with an unexamined # StreamClosedError on the loop). self.server_ioloop.add_callback(self.server_ioloop.stop) self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return 'http://127.0.0.1:%d%s' % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url('/')) self.assertEqual(b'Hello world!', response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url('/notfound')) self.assertEqual(assertion.exception.code, 404)
def periodic(self, data): try: res = data.decode('utf-8') payload = json.loads(res) name = payload.get("name") or "PeriodicTask" method = payload.get("method") timeinterval = payload.get("interval") pt = PeriodicTask(name, payload) self.task_queue.put(pt) loop = IOLoop().current() loop.add_callback(self._process_tasks) except Exception as e: print(f"Cant periodic task {str(e)}", flush=True)
def _initialize(queue): result = None try: # create new IOLoop in the thread io_loop = IOLoop() # make it default for that thread io_loop.make_current() result = io_loop io_loop.add_callback(queue.put, result) io_loop.start() except Exception as err: # pragma: no cover result = err finally: # pragma: no cover queue.put(result)
class TestIOLoopCurrent(unittest.TestCase): def setUp(self): self.io_loop = IOLoop() def tearDown(self): self.io_loop.close() def test_current(self): def f(): self.current_io_loop = IOLoop.current() self.io_loop.stop() self.io_loop.add_callback(f) self.io_loop.start() self.assertIs(self.current_io_loop, self.io_loop)
def safe_shutdown(ioloop: IOLoop, shutdown_function: Callable[[], None]) -> None: def hard_exit() -> None: context_dump(ioloop) sys.stdout.flush() # Hard exit, not sys.exit # ensure shutdown when the ioloop is stuck os._exit(const.EXIT_HARD) # force shutdown, even when the ioloop is stuck # schedule off the loop t = Timer(const.SHUTDOWN_GRACE_HARD, hard_exit) t.daemon = True t.start() ioloop.add_callback(safe_shutdown_wrapper, shutdown_function)
class TestIOLoopCurrent(unittest.TestCase): def setUp(self): self.io_loop = IOLoop() def tearDown(self): self.io_loop.close() def test_current(self): def f(): self.current_io_loop = IOLoop.current() self.io_loop.stop() self.io_loop.add_callback(f) self.io_loop.start() self.assertIs(self.current_io_loop, self.io_loop)
def pdf_capture(static_path, capture_server_class=None): """ Starts a tornado server which serves all of the jupyter path locations as well as the working directory """ settings = { "static_path": static_path } handlers = [ (r"/(.*)", tornado.web.StaticFileHandler, { "path": settings['static_path'] }) ] # add the jupyter static paths for path in jupyter_path(): handlers += [ (r"/static/(.*)", tornado.web.StaticFileHandler, { "path": os.path.join(path, "static") }) ] app = tornado.web.Application(handlers, **settings) if capture_server_class is None: server = CaptureServer(app) else: _module, _klass = capture_server_class.split(":") server = getattr(import_module(_module), _klass)(app) # can't pass this to the constructor for some reason... server.static_path = static_path # add the parsed, normalized notebook with open(os.path.join(static_path, "notebook.ipynb")) as fp: server.notebook = nbformat.read(fp, IPYNB_VERSION) ioloop = IOLoop() # server.capture will be called when the ioloop is bored for the first time ioloop.add_callback(server.capture) # connect to a port server.listen(PORT) try: # run forever ioloop.start() except KeyboardInterrupt: # this is probably not the best way to escape, but works for now print("Successfully created PDF")
class IOLoop(object): NONE = TornadoIOLoop.NONE READ = TornadoIOLoop.READ WRITE = TornadoIOLoop.WRITE ERROR = TornadoIOLoop.ERROR def __init__(self): self._tornado_io_loop = TornadoIOLoop() def inner(self): return self._tornado_io_loop def close(self, all_fds=False): self._tornado_io_loop.close(all_fds) def add_handler(self, fd, handler, events): self._tornado_io_loop.add_handler(fd, handler, events) def update_handler(self, fd, events): self._tornado_io_loop.update_handler(fd, events) def remove_handler(self, fd): self._tornado_io_loop.remove_handler(fd) def start(self): self._tornado_io_loop.start() def stop(self): self._tornado_io_loop.stop() def time(self): return self._tornado_io_loop.time() def add_timeout(self, deadline, callback): return self._tornado_io_loop.add_timeout(deadline, callback) def remove_timeout(self, timeout): self._tornado_io_loop.remove_timeout(timeout) def add_callback(self, callback, *args, **kwargs): self._tornado_io_loop.add_callback(callback, *args, **kwargs) def run(self): try: self.start() except KeyboardInterrupt: print "" print "Ctrl-C recieved. Exiting."
def run_worker(q, ip, port, center_ip, center_port, ncores, nanny_port): """ Function run by the Nanny when creating the worker """ from distributed import Worker from tornado.ioloop import IOLoop IOLoop.clear_instance() loop = IOLoop() loop.make_current() worker = Worker(ip, port, center_ip, center_port, ncores, nanny_port=nanny_port) @gen.coroutine def start(): yield worker._start() q.put(worker.port) loop.add_callback(start) loop.start()
class TestIOLoopCurrent(unittest.TestCase): def setUp(self): setup_with_context_manager(self, ignore_deprecation()) self.io_loop = None # type: typing.Optional[IOLoop] IOLoop.clear_current() def tearDown(self): if self.io_loop is not None: self.io_loop.close() def test_default_current(self): self.io_loop = IOLoop() # The first IOLoop with default arguments is made current. self.assertIs(self.io_loop, IOLoop.current()) # A second IOLoop can be created but is not made current. io_loop2 = IOLoop() self.assertIs(self.io_loop, IOLoop.current()) io_loop2.close() def test_non_current(self): self.io_loop = IOLoop(make_current=False) # The new IOLoop is not initially made current. self.assertIsNone(IOLoop.current(instance=False)) # Starting the IOLoop makes it current, and stopping the loop # makes it non-current. This process is repeatable. for i in range(3): def f(): self.current_io_loop = IOLoop.current() assert self.io_loop is not None self.io_loop.stop() self.io_loop.add_callback(f) self.io_loop.start() self.assertIs(self.current_io_loop, self.io_loop) # Now that the loop is stopped, it is no longer current. self.assertIsNone(IOLoop.current(instance=False)) def test_force_current(self): self.io_loop = IOLoop(make_current=True) self.assertIs(self.io_loop, IOLoop.current()) with self.assertRaises(RuntimeError): # A second make_current=True construction cannot succeed. IOLoop(make_current=True) # current() was not affected by the failed construction. self.assertIs(self.io_loop, IOLoop.current())
def test_add_callback_while_closing(self): # add_callback should not fail if it races with another thread # closing the IOLoop. The callbacks are dropped silently # without executing. closing = threading.Event() def target(): other_ioloop.add_callback(other_ioloop.stop) other_ioloop.start() closing.set() other_ioloop.close(all_fds=True) other_ioloop = IOLoop() thread = threading.Thread(target=target) thread.start() closing.wait() for i in range(1000): other_ioloop.add_callback(lambda: None)
def set_tasks(self, io_loop: IOLoop): """ Set server callbacks on given IO loop. Must be called once per server before starting IO loop. """ io_loop.add_callback(self._task_save_database) io_loop.add_callback(self._task_send_notifications) # These both coroutines are used to manage games. io_loop.add_callback(self.games_scheduler.process_tasks) io_loop.add_callback(self.games_scheduler.schedule) # Set callback on KeyboardInterrupt. signal.signal(signal.SIGINT, self.interruption_handler.handler) atexit.register(self.backup_now)
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ in ("TwistedIOLoop", "AsyncIOMainLoop"): # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. # AsyncIOMainLoop doesn't work with the default policy # (although it could with some tweaks to this test and a # policy that created loops for non-main threads). raise unittest.SkipTest("Sync HTTPClient not compatible with TwistedIOLoop or " "AsyncIOMainLoop") self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([("/", HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() self.server_ioloop.stop() self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return "http://127.0.0.1:%d%s" % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url("/")) self.assertEqual(b"Hello world!", response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url("/notfound")) self.assertEqual(assertion.exception.code, 404)
class ProxyTestServer(object): def __init__(self,): self.server_ioloop = IOLoop() self.access_count = 0 @tornado.gen.coroutine def init_server(): sock, self.port = bind_unused_port() app = Application([('/(.*)', ProxyTestHandler, dict(server=self))]) self.server = HTTPServer(app) self.server.add_socket(sock) self.server_ioloop.run_sync(init_server) self.server_thread = threading.Thread(target=self.server_ioloop.start) def start(self): self.server_thread.start() def stop(self): def stop_server(): self.server.stop() @tornado.gen.coroutine def slow_stop(): for i in range(5): yield self.server_ioloop.stop() self.server_ioloop.add_callback(slow_stop) self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.server_ioloop.close(all_fds=True) def get_access_count(self): return self.access_count def clear_access_count(self): self.access_count = 0
def pdf_capture(static_path, capture_server_class=None): """ Starts a tornado server which serves all of the jupyter path locations as well as the working directory """ settings = {"static_path": static_path} handlers = [(r"/(.*)", tornado.web.StaticFileHandler, { "path": settings['static_path'] })] # add the jupyter static paths for path in jupyter_path(): handlers += [(r"/static/(.*)", tornado.web.StaticFileHandler, { "path": os.path.join(path, "static") })] app = tornado.web.Application(handlers, **settings) if capture_server_class is None: server = CaptureServer(app) else: _module, _klass = capture_server_class.split(":") server = getattr(import_module(_module), _klass)(app) # can't pass this to the constructor for some reason... server.static_path = static_path # add the parsed, normalized notebook with open(os.path.join(static_path, "notebook.ipynb")) as fp: server.notebook = nbformat.read(fp, IPYNB_VERSION) ioloop = IOLoop() # server.capture will be called when the ioloop is bored for the first time ioloop.add_callback(server.capture) # connect to a port server.listen(PORT) try: # run forever ioloop.start() except KeyboardInterrupt: # this is probably not the best way to escape, but works for now print("Successfully created PDF")
class LoopOverhead(object): """ These are not distributed benchmarks per se, but help assessing Tornado's loop management overhead for other benchmarks. """ def setup(self): self.loop = IOLoop() self.loop.make_current() def time_loop_start_stop(self): self.loop.add_callback(self.loop.stop) self.loop.start() @gen.coroutine def _empty_coro(self): pass def time_loop_run_sync(self): run_sync(self.loop, self._empty_coro)
def test_add_callback_while_closing(self): # Issue #635: add_callback() should raise a clean exception # if called while another thread is closing the IOLoop. closing = threading.Event() def target(): other_ioloop.add_callback(other_ioloop.stop) other_ioloop.start() closing.set() other_ioloop.close(all_fds=True) other_ioloop = IOLoop() thread = threading.Thread(target=target) thread.start() closing.wait() for i in range(1000): try: other_ioloop.add_callback(lambda: None) except RuntimeError, e: self.assertEqual("IOLoop is closing", str(e)) break
class TestIOLoopCurrent(unittest.TestCase): def setUp(self): self.io_loop = None IOLoop.clear_current() def tearDown(self): if self.io_loop is not None: self.io_loop.close() def test_default_current(self): self.io_loop = IOLoop() # The first IOLoop with default arguments is made current. self.assertIs(self.io_loop, IOLoop.current()) # A second IOLoop can be created but is not made current. io_loop2 = IOLoop() self.assertIs(self.io_loop, IOLoop.current()) io_loop2.close() def test_non_current(self): self.io_loop = IOLoop(make_current=False) # The new IOLoop is not initially made current. self.assertIsNone(IOLoop.current(instance=False)) # Starting the IOLoop makes it current, and stopping the loop # makes it non-current. This process is repeatable. for i in range(3): def f(): self.current_io_loop = IOLoop.current() self.io_loop.stop() self.io_loop.add_callback(f) self.io_loop.start() self.assertIs(self.current_io_loop, self.io_loop) # Now that the loop is stopped, it is no longer current. self.assertIsNone(IOLoop.current(instance=False)) def test_force_current(self): self.io_loop = IOLoop(make_current=True) self.assertIs(self.io_loop, IOLoop.current()) with self.assertRaises(RuntimeError): # A second make_current=True construction cannot succeed. IOLoop(make_current=True) # current() was not affected by the failed construction. self.assertIs(self.io_loop, IOLoop.current())
class ProxyTestServer(object): def __init__(self, ): self.server_ioloop = IOLoop() self.access_count = 0 @tornado.gen.coroutine def init_server(): sock, self.port = bind_unused_port() app = Application([('/(.*)', ProxyTestHandler, dict(server=self))]) self.server = HTTPServer(app) self.server.add_socket(sock) self.server_ioloop.run_sync(init_server) self.server_thread = threading.Thread(target=self.server_ioloop.start) def start(self): self.server_thread.start() def stop(self): def stop_server(): self.server.stop() @tornado.gen.coroutine def slow_stop(): for i in range(5): yield self.server_ioloop.stop() self.server_ioloop.add_callback(slow_stop) self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.server_ioloop.close(all_fds=True) def get_access_count(self): return self.access_count def clear_access_count(self): self.access_count = 0
class Stream(threading.Thread): def __init__(self, stream_ip, stream_name): super().__init__() # set ip and name self._ip = stream_ip self._name = stream_name # set logger self._logger = Logger('stream') # event loop self._loop = IOLoop() def run(self): # queue main action self._loop.add_callback(self._once) # start event loop in this thread self._loop.start() # free all resources self._loop.close() @coroutine def stop(self): ok = yield Task( lambda callback: self._loop.add_callback(self._stop_, callback)) return ok def _once(self): # read and then write rtmp stream (for hls feature) cmd = 'ffmpeg -i rtmp://{0}/src/{1} -c:a aac -b:a 64k -c:v libx264 -b:v 256k -vf scale=-1:480 -f flv rtmp://{0}/hls/{1}_480p -c:a aac -b:a 128k -c:v libx264 -b:v 512K -vf scale=-1:720 -f flv rtmp://{0}/hls/{1}_720p'.format( self._ip, self._name) self._child = Subprocess(shlex.split(cmd), stdout=Subprocess.STREAM, stderr=Subprocess.STREAM) # stop loop self._loop.stop() def _stop(self): try: pid = self._child.proc.pid self._logger.debug('_stop') os.kill(pid, signal.SIGTERM) except ProcessLookupError as e: self._logger.exception('error on killing pid {0}'.format(pid))
class TestIOLoopCurrent(unittest.TestCase): def setUp(self): self.io_loop = None IOLoop.clear_current() def tearDown(self): if self.io_loop is not None: self.io_loop.close() def test_default_current(self): self.io_loop = IOLoop() # The first IOLoop with default arguments is made current. self.assertIs(self.io_loop, IOLoop.current()) # A second IOLoop can be created but is not made current. io_loop2 = IOLoop() self.assertIs(self.io_loop, IOLoop.current()) io_loop2.close() def test_non_current(self): self.io_loop = IOLoop(make_current=False) # The new IOLoop is not initially made current. self.assertIsNone(IOLoop.current(instance=False)) def f(): # But it is current after it is started. self.current_io_loop = IOLoop.current() self.io_loop.stop() self.io_loop.add_callback(f) self.io_loop.start() self.assertIs(self.current_io_loop, self.io_loop) # Now that the loop is stopped, it is no longer current. self.assertIsNone(IOLoop.current(instance=False)) def test_force_current(self): self.io_loop = IOLoop(make_current=True) self.assertIs(self.io_loop, IOLoop.current()) with self.assertRaises(RuntimeError): # A second make_current=True construction cannot succeed. IOLoop(make_current=True) # current() was not affected by the failed construction. self.assertIs(self.io_loop, IOLoop.current())
class ProxyTestServer(object): """ Tornado test server for use in proxy testing.""" def __init__(self, ): """Initialise the server.""" self.access_count = 0 self.server_event_loop = None self.server_thread = threading.Thread(target=self._run_server) self.server_thread.start() time.sleep(0.2) def _run_server(self): if sys.version_info[0] == 3: asyncio.set_event_loop(asyncio.new_event_loop()) self.server_event_loop = IOLoop() self.sock, self.port = bind_unused_port() self.app = Application([('/(.*)', ProxyTestHandler, dict(server=self)) ]) self.server = HTTPServer(self.app) self.server.add_socket(self.sock) self.server_event_loop.start() def stop(self): """Stop the server, using a callback added to the server IOLoop.""" if self.server_thread is not None: self.server_event_loop.add_callback(self.server_event_loop.stop) self.server_thread.join() self.server_thread = None self.server.stop() def get_access_count(self): """Return the server access count.""" return self.access_count def clear_access_count(self): """Clear the server access count.""" self.access_count = 0
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ == 'TwistedIOLoop': # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop') self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() self.server_ioloop.stop() self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return 'http://localhost:%d%s' % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url('/')) self.assertEqual(b'Hello world!', response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url('/notfound')) self.assertEqual(assertion.exception.code, 404)
def test_add_callback_while_closing(self): # Issue #635: add_callback() should raise a clean exception # if called while another thread is closing the IOLoop. closing = threading.Event() def target(): other_ioloop.add_callback(other_ioloop.stop) other_ioloop.start() closing.set() other_ioloop.close(all_fds=True) other_ioloop = IOLoop() thread = threading.Thread(target=target) thread.start() closing.wait() for i in range(1000): try: other_ioloop.add_callback(lambda: None) except RuntimeError as e: self.assertEqual("IOLoop is closing", str(e)) break
class ControlThread(Thread): def __init__(self, **kwargs): Thread.__init__(self, name="Control", **kwargs) self.io_loop = IOLoop(make_current=False) self.pydev_do_not_trace = True self.is_pydev_daemon_thread = True def run(self): self.name = "Control" try: self.io_loop.start() finally: self.io_loop.close() def stop(self): """Stop the thread. This method is threadsafe. """ self.io_loop.add_callback(self.io_loop.stop)
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ == 'TwistedIOLoop': # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop') self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() self.server_ioloop.stop() self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return 'http://localhost:%d%s' % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url('/')) self.assertEqual(b'Hello world!', response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url('/notfound')) self.assertEqual(assertion.exception.code, 404)
def test_add_callback_while_closing(self): # Issue #635: add_callback() should raise a clean exception # if called while another thread is closing the IOLoop. if IOLoop.configured_class().__name__.endswith('AsyncIOLoop'): raise unittest.SkipTest("AsyncIOMainLoop shutdown not thread safe") closing = threading.Event() def target(): other_ioloop.add_callback(other_ioloop.stop) other_ioloop.start() closing.set() other_ioloop.close(all_fds=True) other_ioloop = IOLoop() thread = threading.Thread(target=target) thread.start() closing.wait() for i in range(1000): try: other_ioloop.add_callback(lambda: None) except RuntimeError as e: self.assertEqual("IOLoop is closing", str(e)) break
class Server(BaseServer): """ Thread wrapper over `tornado.transport.server.Server` """ def __init__(self, handler): self.io_loop = IOLoop() super(Server, self).__init__(handler, io_loop=self.io_loop) self._serving_thread = None def listen(self): self._serving_thread = threading.Thread(target=self._listen) self._serving_thread.start() def _listen(self): super(Server, self).listen() self.io_loop.start() def stop(self): self.io_loop.add_callback(super(Server, self).stop) self.io_loop.add_callback(self.io_loop.stop) def handle(self, method, *args, **kwargs): return ServerTransport.handle(self, method, *args, **kwargs)
def test__ioloop_not_forcibly_stopped(): # Issue #5494 application = Application() loop = IOLoop() loop.make_current() server = Server(application, io_loop=loop) server.start() result = [] def f(): server.unlisten() server.stop() # If server.stop() were to stop the Tornado IO loop, # g() wouldn't be called and `result` would remain empty. loop.add_timeout(timedelta(seconds=0.01), g) def g(): result.append(None) loop.stop() loop.add_callback(f) loop.start() assert result == [None]
def run(self): if not self._allowRoot: self._check_for_root() global app global babel global printer global printerProfileManager global fileManager global slicingManager global analysisQueue global userManager global eventManager global loginManager global pluginManager global appSessionManager global pluginLifecycleManager global preemptiveCache global debug from tornado.ioloop import IOLoop from tornado.web import Application, RequestHandler import sys debug = self._debug # first initialize the settings singleton and make sure it uses given configfile and basedir if available s = settings(init=True, basedir=self._basedir, configfile=self._configfile) # then monkey patch a bunch of stuff util.tornado.fix_ioloop_scheduling() util.flask.enable_additional_translations(additional_folders=[s.getBaseFolder("translations")]) # setup app self._setup_app(app) # setup i18n self._setup_i18n(app) # then initialize logging self._setup_logging(self._debug, self._logConf) self._logger = logging.getLogger(__name__) def exception_logger(exc_type, exc_value, exc_tb): self._logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_tb)) sys.excepthook = exception_logger self._logger.info("Starting OctoPrint %s" % DISPLAY_VERSION) # start the intermediary server self._start_intermediary_server(s) # then initialize the plugin manager pluginManager = octoprint.plugin.plugin_manager(init=True) printerProfileManager = PrinterProfileManager() eventManager = events.eventManager() analysisQueue = octoprint.filemanager.analysis.AnalysisQueue() slicingManager = octoprint.slicing.SlicingManager(s.getBaseFolder("slicingProfiles"), printerProfileManager) storage_managers = dict() storage_managers[octoprint.filemanager.FileDestinations.LOCAL] = octoprint.filemanager.storage.LocalFileStorage(s.getBaseFolder("uploads")) fileManager = octoprint.filemanager.FileManager(analysisQueue, slicingManager, printerProfileManager, initial_storage_managers=storage_managers) printer = Printer(fileManager, analysisQueue, printerProfileManager) appSessionManager = util.flask.AppSessionManager() pluginLifecycleManager = LifecycleManager(pluginManager) preemptiveCache = PreemptiveCache(os.path.join(s.getBaseFolder("data"), "preemptive_cache_config.yaml")) # ... and initialize all plugins def octoprint_plugin_inject_factory(name, implementation): """Factory for injections for all OctoPrintPlugins""" if not isinstance(implementation, octoprint.plugin.OctoPrintPlugin): # we only care about OctoPrintPlugins return None return dict( plugin_manager=pluginManager, printer_profile_manager=printerProfileManager, event_bus=eventManager, analysis_queue=analysisQueue, slicing_manager=slicingManager, file_manager=fileManager, printer=printer, app_session_manager=appSessionManager, plugin_lifecycle_manager=pluginLifecycleManager, data_folder=os.path.join(settings().getBaseFolder("data"), name), preemptive_cache=preemptiveCache ) def settings_plugin_inject_factory(name, implementation): """Factory for additional injections depending on plugin type""" if not isinstance(implementation, octoprint.plugin.SettingsPlugin): # we only care about SettingsPlugins return None # SettingsPlugin instnances get a PluginSettings instance injected default_settings = implementation.get_settings_defaults() get_preprocessors, set_preprocessors = implementation.get_settings_preprocessors() plugin_settings = octoprint.plugin.plugin_settings(name, defaults=default_settings, get_preprocessors=get_preprocessors, set_preprocessors=set_preprocessors) return dict(settings=plugin_settings) def settings_plugin_config_migration_and_cleanup(name, implementation): """Take care of migrating and cleaning up any old settings""" if not isinstance(implementation, octoprint.plugin.SettingsPlugin): return settings_version = implementation.get_settings_version() settings_migrator = implementation.on_settings_migrate if settings_version is not None and settings_migrator is not None: stored_version = implementation._settings.get_int([octoprint.plugin.SettingsPlugin.config_version_key]) if stored_version is None or stored_version < settings_version: settings_migrator(settings_version, stored_version) implementation._settings.set_int([octoprint.plugin.SettingsPlugin.config_version_key], settings_version) implementation.on_settings_cleanup() implementation._settings.save() implementation.on_settings_initialized() pluginManager.implementation_inject_factories=[octoprint_plugin_inject_factory, settings_plugin_inject_factory] pluginManager.initialize_implementations() settingsPlugins = pluginManager.get_implementations(octoprint.plugin.SettingsPlugin) for implementation in settingsPlugins: try: settings_plugin_config_migration_and_cleanup(implementation._identifier, implementation) except: self._logger.exception("Error while trying to migrate settings for plugin {}, ignoring it".format(implementation._identifier)) pluginManager.implementation_post_inits=[settings_plugin_config_migration_and_cleanup] pluginManager.log_all_plugins() # initialize file manager and register it for changes in the registered plugins fileManager.initialize() pluginLifecycleManager.add_callback(["enabled", "disabled"], lambda name, plugin: fileManager.reload_plugins()) # initialize slicing manager and register it for changes in the registered plugins slicingManager.initialize() pluginLifecycleManager.add_callback(["enabled", "disabled"], lambda name, plugin: slicingManager.reload_slicers()) # setup jinja2 self._setup_jinja2() # make sure plugin lifecycle events relevant for jinja2 are taken care of def template_enabled(name, plugin): if plugin.implementation is None or not isinstance(plugin.implementation, octoprint.plugin.TemplatePlugin): return self._register_additional_template_plugin(plugin.implementation) def template_disabled(name, plugin): if plugin.implementation is None or not isinstance(plugin.implementation, octoprint.plugin.TemplatePlugin): return self._unregister_additional_template_plugin(plugin.implementation) pluginLifecycleManager.add_callback("enabled", template_enabled) pluginLifecycleManager.add_callback("disabled", template_disabled) # setup assets self._setup_assets() # configure timelapse octoprint.timelapse.configure_timelapse() # setup command triggers events.CommandTrigger(printer) if self._debug: events.DebugEventListener() # setup access control userManagerName = s.get(["accessControl", "userManager"]) try: clazz = octoprint.util.get_class(userManagerName) userManager = clazz() except AttributeError as e: self._logger.exception("Could not instantiate user manager {}, falling back to FilebasedUserManager!".format(userManagerName)) userManager = octoprint.users.FilebasedUserManager() finally: userManager.enabled = s.getBoolean(["accessControl", "enabled"]) loginManager = LoginManager() loginManager.session_protection = "strong" loginManager.user_callback = load_user if not userManager.enabled: loginManager.anonymous_user = users.DummyUser principals.identity_loaders.appendleft(users.dummy_identity_loader) loginManager.init_app(app) # register API blueprint self._setup_blueprints() ## Tornado initialization starts here if self._host is None: self._host = s.get(["server", "host"]) if self._port is None: self._port = s.getInt(["server", "port"]) ioloop = IOLoop() ioloop.install() self._router = SockJSRouter(self._create_socket_connection, "/sockjs") upload_suffixes = dict(name=s.get(["server", "uploads", "nameSuffix"]), path=s.get(["server", "uploads", "pathSuffix"])) def mime_type_guesser(path): from octoprint.filemanager import get_mime_type return get_mime_type(path) download_handler_kwargs = dict( as_attachment=True, allow_client_caching=False ) additional_mime_types=dict(mime_type_guesser=mime_type_guesser) admin_validator = dict(access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.admin_validator)) no_hidden_files_validator = dict(path_validation=util.tornado.path_validation_factory(lambda path: not octoprint.util.is_hidden_path(path), status_code=404)) def joined_dict(*dicts): if not len(dicts): return dict() joined = dict() for d in dicts: joined.update(d) return joined server_routes = self._router.urls + [ # various downloads (r"/downloads/timelapse/([^/]*\.mp[g4])", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("timelapse")), download_handler_kwargs, no_hidden_files_validator)), (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("uploads")), download_handler_kwargs, no_hidden_files_validator, additional_mime_types)), (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("logs")), download_handler_kwargs, admin_validator)), # camera snapshot (r"/downloads/camera/current", util.tornado.UrlProxyHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), # generated webassets (r"/static/webassets/(.*)", util.tornado.LargeResponseHandler, dict(path=os.path.join(s.getBaseFolder("generated"), "webassets"))), # online indicators - text file with "online" as content and a transparent gif (r"/online.txt", util.tornado.StaticDataHandler, dict(data="online\n")), (r"/online.gif", util.tornado.StaticDataHandler, dict(data=bytes(base64.b64decode("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7")), content_type="image/gif")) ] # fetch additional routes from plugins for name, hook in pluginManager.get_hooks("octoprint.server.http.routes").items(): try: result = hook(list(server_routes)) except: self._logger.exception("There was an error while retrieving additional server routes from plugin hook {name}".format(**locals())) else: if isinstance(result, (list, tuple)): for entry in result: if not isinstance(entry, tuple) or not len(entry) == 3: continue if not isinstance(entry[0], basestring): continue if not isinstance(entry[2], dict): continue route, handler, kwargs = entry route = r"/plugin/{name}/{route}".format(name=name, route=route if not route.startswith("/") else route[1:]) self._logger.debug("Adding additional route {route} handled by handler {handler} and with additional arguments {kwargs!r}".format(**locals())) server_routes.append((route, handler, kwargs)) server_routes.append((r".*", util.tornado.UploadStorageFallbackHandler, dict(fallback=util.tornado.WsgiInputContainer(app.wsgi_app), file_prefix="octoprint-file-upload-", file_suffix=".tmp", suffixes=upload_suffixes))) self._tornado_app = Application(server_routes) max_body_sizes = [ ("POST", r"/api/files/([^/]*)", s.getInt(["server", "uploads", "maxSize"])), ("POST", r"/api/languages", 5 * 1024 * 1024) ] # allow plugins to extend allowed maximum body sizes for name, hook in pluginManager.get_hooks("octoprint.server.http.bodysize").items(): try: result = hook(list(max_body_sizes)) except: self._logger.exception("There was an error while retrieving additional upload sizes from plugin hook {name}".format(**locals())) else: if isinstance(result, (list, tuple)): for entry in result: if not isinstance(entry, tuple) or not len(entry) == 3: continue if not entry[0] in util.tornado.UploadStorageFallbackHandler.BODY_METHODS: continue if not isinstance(entry[2], int): continue method, route, size = entry route = r"/plugin/{name}/{route}".format(name=name, route=route if not route.startswith("/") else route[1:]) self._logger.debug("Adding maximum body size of {size}B for {method} requests to {route})".format(**locals())) max_body_sizes.append((method, route, size)) self._stop_intermediary_server() # initialize and bind the server self._server = util.tornado.CustomHTTPServer(self._tornado_app, max_body_sizes=max_body_sizes, default_max_body_size=s.getInt(["server", "maxSize"])) self._server.listen(self._port, address=self._host) eventManager.fire(events.Events.STARTUP) # auto connect if s.getBoolean(["serial", "autoconnect"]): (port, baudrate) = s.get(["serial", "port"]), s.getInt(["serial", "baudrate"]) printer_profile = printerProfileManager.get_default() connectionOptions = get_connection_options() if port in connectionOptions["ports"]: printer.connect(port=port, baudrate=baudrate, profile=printer_profile["id"] if "id" in printer_profile else "_default") # start up watchdogs if s.getBoolean(["feature", "pollWatched"]): # use less performant polling observer if explicitely configured observer = PollingObserver() else: # use os default observer = Observer() observer.schedule(util.watchdog.GcodeWatchdogHandler(fileManager, printer), s.getBaseFolder("watched")) observer.start() # run our startup plugins octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin, "on_startup", args=(self._host, self._port)) def call_on_startup(name, plugin): implementation = plugin.get_implementation(octoprint.plugin.StartupPlugin) if implementation is None: return implementation.on_startup(self._host, self._port) pluginLifecycleManager.add_callback("enabled", call_on_startup) # prepare our after startup function def on_after_startup(): self._logger.info("Listening on http://%s:%d" % (self._host, self._port)) # now this is somewhat ugly, but the issue is the following: startup plugins might want to do things for # which they need the server to be already alive (e.g. for being able to resolve urls, such as favicons # or service xmls or the like). While they are working though the ioloop would block. Therefore we'll # create a single use thread in which to perform our after-startup-tasks, start that and hand back # control to the ioloop def work(): octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin, "on_after_startup") def call_on_after_startup(name, plugin): implementation = plugin.get_implementation(octoprint.plugin.StartupPlugin) if implementation is None: return implementation.on_after_startup() pluginLifecycleManager.add_callback("enabled", call_on_after_startup) # when we are through with that we also run our preemptive cache if settings().getBoolean(["devel", "cache", "preemptive"]): self._execute_preemptive_flask_caching(preemptiveCache) import threading threading.Thread(target=work).start() ioloop.add_callback(on_after_startup) # prepare our shutdown function def on_shutdown(): # will be called on clean system exit and shutdown the watchdog observer and call the on_shutdown methods # on all registered ShutdownPlugins self._logger.info("Shutting down...") observer.stop() observer.join() octoprint.plugin.call_plugin(octoprint.plugin.ShutdownPlugin, "on_shutdown") if self._octoprint_daemon is not None: self._logger.info("Cleaning up daemon pidfile") self._octoprint_daemon.terminated() self._logger.info("Goodbye!") atexit.register(on_shutdown) def sigterm_handler(*args, **kwargs): # will stop tornado on SIGTERM, making the program exit cleanly def shutdown_tornado(): ioloop.stop() ioloop.add_callback_from_signal(shutdown_tornado) signal.signal(signal.SIGTERM, sigterm_handler) try: # this is the main loop - as long as tornado is running, OctoPrint is running ioloop.start() except (KeyboardInterrupt, SystemExit): pass except: self._logger.fatal("Now that is embarrassing... Something really really went wrong here. Please report this including the stacktrace below in OctoPrint's bugtracker. Thanks!") self._logger.exception("Stacktrace follows:")
class MockHub(JupyterHub): """Hub with various mock bits""" db_file = None def _ip_default(self): return localhost() def _authenticator_class_default(self): return MockPAMAuthenticator def _spawner_class_default(self): return MockSpawner def init_signal(self): pass def start(self, argv=None): self.db_file = NamedTemporaryFile() self.db_url = 'sqlite:///' + self.db_file.name evt = threading.Event() @gen.coroutine def _start_co(): assert self.io_loop._running # put initialize in start for SQLAlchemy threading reasons yield super(MockHub, self).initialize(argv=argv) # add an initial user user = orm.User(name='user') self.db.add(user) self.db.commit() yield super(MockHub, self).start() yield self.hub.server.wait_up(http=True) self.io_loop.add_callback(evt.set) def _start(): self.io_loop = IOLoop() self.io_loop.make_current() self.io_loop.add_callback(_start_co) self.io_loop.start() self._thread = threading.Thread(target=_start) self._thread.start() ready = evt.wait(timeout=10) assert ready def stop(self): super().stop() self._thread.join() IOLoop().run_sync(self.cleanup) # ignore the call that will fire in atexit self.cleanup = lambda : None self.db_file.close() def login_user(self, name): r = requests.post(self.proxy.public_server.url + 'hub/login', data={ 'username': name, 'password': name, }, allow_redirects=False, ) assert r.cookies return r.cookies
class LoopRunner(object): """ A helper to start and stop an IO loop in a controlled way. Several loop runners can associate safely to the same IO loop. Parameters ---------- loop: IOLoop (optional) If given, this loop will be re-used, otherwise an appropriate one will be looked up or created. asynchronous: boolean (optional, default False) If false (the default), the loop is meant to run in a separate thread and will be started if necessary. If true, the loop is meant to run in the thread this object is instantiated from, and will not be started automatically. """ # All loops currently associated to loop runners _all_loops = weakref.WeakKeyDictionary() _lock = threading.Lock() def __init__(self, loop=None, asynchronous=False): current = IOLoop.current() if loop is None: if asynchronous: self._loop = current else: # We're expecting the loop to run in another thread, # avoid re-using this thread's assigned loop self._loop = IOLoop() self._should_close_loop = True else: self._loop = loop self._should_close_loop = False self._asynchronous = asynchronous self._loop_thread = None self._started = False with self._lock: self._all_loops.setdefault(self._loop, (0, None)) def start(self): """ Start the IO loop if required. The loop is run in a dedicated thread. If the loop is already running, this method does nothing. """ with self._lock: self._start_unlocked() def _start_unlocked(self): assert not self._started count, real_runner = self._all_loops[self._loop] if (self._asynchronous or real_runner is not None or count > 0): self._all_loops[self._loop] = count + 1, real_runner self._started = True return assert self._loop_thread is None assert count == 0 loop_evt = threading.Event() done_evt = threading.Event() in_thread = [None] start_exc = [None] def loop_cb(): in_thread[0] = threading.current_thread() loop_evt.set() def run_loop(loop=self._loop): loop.add_callback(loop_cb) try: loop.start() except Exception as e: start_exc[0] = e finally: done_evt.set() thread = threading.Thread(target=run_loop, name="IO loop") thread.daemon = True thread.start() loop_evt.wait(timeout=10) self._started = True actual_thread = in_thread[0] if actual_thread is not thread: # Loop already running in other thread (user-launched) done_evt.wait(5) if not isinstance(start_exc[0], RuntimeError): if not isinstance(start_exc[0], Exception): # track down infrequent error raise TypeError("not an exception", start_exc[0]) raise start_exc[0] self._all_loops[self._loop] = count + 1, None else: assert start_exc[0] is None, start_exc self._loop_thread = thread self._all_loops[self._loop] = count + 1, self def stop(self, timeout=10): """ Stop and close the loop if it was created by us. Otherwise, just mark this object "stopped". """ with self._lock: self._stop_unlocked(timeout) def _stop_unlocked(self, timeout): if not self._started: return self._started = False count, real_runner = self._all_loops[self._loop] if count > 1: self._all_loops[self._loop] = count - 1, real_runner else: assert count == 1 del self._all_loops[self._loop] if real_runner is not None: real_runner._real_stop(timeout) def _real_stop(self, timeout): assert self._loop_thread is not None if self._loop_thread is not None: try: self._loop.add_callback(self._loop.stop) self._loop_thread.join(timeout=timeout) self._loop.close() finally: self._loop_thread = None def is_started(self): """ Return True between start() and stop() calls, False otherwise. """ return self._started def run_sync(self, func, *args, **kwargs): """ Convenience helper: start the loop if needed, run sync(func, *args, **kwargs), then stop the loop again. """ if self._started: return sync(self.loop, func, *args, **kwargs) else: self.start() try: return sync(self.loop, func, *args, **kwargs) finally: self.stop() @property def loop(self): return self._loop
class CompatibilityTests(unittest.TestCase): def setUp(self): self.saved_signals = save_signal_handlers() self.io_loop = IOLoop() self.io_loop.make_current() self.reactor = AsyncioSelectorReactor() def tearDown(self): self.reactor.disconnectAll() self.io_loop.clear_current() self.io_loop.close(all_fds=True) restore_signal_handlers(self.saved_signals) def start_twisted_server(self): class HelloResource(Resource): isLeaf = True def render_GET(self, request): return b"Hello from twisted!" site = Site(HelloResource()) port = self.reactor.listenTCP(0, site, interface='127.0.0.1') self.twisted_port = port.getHost().port def start_tornado_server(self): class HelloHandler(RequestHandler): def get(self): self.write("Hello from tornado!") app = Application([('/', HelloHandler)], log_function=lambda x: None) server = HTTPServer(app) sock, self.tornado_port = bind_unused_port() server.add_sockets([sock]) def run_reactor(self): # In theory, we can run the event loop through Tornado, # Twisted, or asyncio interfaces. However, since we're trying # to avoid installing anything as the global event loop, only # the twisted interface gets everything wired up correectly # without extra hacks. This method is a part of a # no-longer-used generalization that allowed us to test # different combinations. self.stop_loop = self.reactor.stop self.stop = self.reactor.stop self.reactor.run() def tornado_fetch(self, url, runner): client = AsyncHTTPClient() fut = client.fetch(url) fut.add_done_callback(lambda f: self.stop_loop()) runner() return fut.result() def twisted_fetch(self, url, runner): # http://twistedmatrix.com/documents/current/web/howto/client.html chunks = [] client = Agent(self.reactor) d = client.request(b'GET', utf8(url)) class Accumulator(Protocol): def __init__(self, finished): self.finished = finished def dataReceived(self, data): chunks.append(data) def connectionLost(self, reason): self.finished.callback(None) def callback(response): finished = Deferred() response.deliverBody(Accumulator(finished)) return finished d.addCallback(callback) def shutdown(failure): if hasattr(self, 'stop_loop'): self.stop_loop() elif failure is not None: # loop hasn't been initialized yet; try our best to # get an error message out. (the runner() interaction # should probably be refactored). try: failure.raiseException() except: logging.error('exception before starting loop', exc_info=True) d.addBoth(shutdown) runner() self.assertTrue(chunks) return b''.join(chunks) def twisted_coroutine_fetch(self, url, runner): body = [None] @gen.coroutine def f(): # This is simpler than the non-coroutine version, but it cheats # by reading the body in one blob instead of streaming it with # a Protocol. client = Agent(self.reactor) response = yield client.request(b'GET', utf8(url)) with warnings.catch_warnings(): # readBody has a buggy DeprecationWarning in Twisted 15.0: # https://twistedmatrix.com/trac/changeset/43379 warnings.simplefilter('ignore', category=DeprecationWarning) body[0] = yield readBody(response) self.stop_loop() self.io_loop.add_callback(f) runner() return body[0] def testTwistedServerTornadoClientReactor(self): self.start_twisted_server() response = self.tornado_fetch( 'http://127.0.0.1:%d' % self.twisted_port, self.run_reactor) self.assertEqual(response.body, b'Hello from twisted!') def testTornadoServerTwistedClientReactor(self): self.start_tornado_server() response = self.twisted_fetch( 'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor) self.assertEqual(response, b'Hello from tornado!') def testTornadoServerTwistedCoroutineClientReactor(self): self.start_tornado_server() response = self.twisted_coroutine_fetch( 'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor) self.assertEqual(response, b'Hello from tornado!')
class KafkaRESTClient(object): def __init__(self, host, port, http_max_clients=10, max_queue_size_per_topic=5000, flush_length_threshold=20, flush_time_threshold_seconds=20, flush_max_batch_size=50, connect_timeout_seconds=10, request_timeout_seconds=60, retry_base_seconds=2, retry_max_attempts=10, retry_period_seconds=15, response_5xx_circuit_breaker_trip_threshold=10, response_5xx_circuit_breaker_trip_duration_seconds=300, shutdown_timeout_seconds=2): self.host = host self.port = port self.http_max_clients = http_max_clients self.max_queue_size_per_topic = max_queue_size_per_topic self.flush_length_threshold = flush_length_threshold self.flush_time_threshold_seconds = flush_time_threshold_seconds self.flush_max_batch_size = flush_max_batch_size self.connect_timeout_seconds = connect_timeout_seconds self.request_timeout_seconds = request_timeout_seconds self.retry_base_seconds = retry_base_seconds # Includes the original send as an attempt, so set to 1 to disable retry self.retry_max_attempts = retry_max_attempts self.retry_period_seconds = retry_period_seconds # Circuit breaker prevents thrashing if we receive multiple transport # errors in a row, since this usually means we are experiencing some # sort of network problem and other requests are very likely to fail. # When this triggers, we wait a short duration before clearing the # breaker and attempting any further network operations. if response_5xx_circuit_breaker_trip_threshold is None: response_5xx_circuit_breaker_trip_threshold = sys.maxsize self.response_5xx_circuit_breaker_trip_threshold = response_5xx_circuit_breaker_trip_threshold self.response_5xx_circuit_breaker_trip_duration_seconds = response_5xx_circuit_breaker_trip_duration_seconds # On shutdown, last-ditch flush attempts are given this # request timeout after which they are considered failed self.shutdown_timeout_seconds = shutdown_timeout_seconds self.in_shutdown = False self.registrar = EventRegistrar() self.response_5xx_circuit_breaker = CircuitBreaker(self.response_5xx_circuit_breaker_trip_threshold, self.response_5xx_circuit_breaker_trip_duration_seconds) self.message_queues = defaultdict(lambda: Queue(maxsize=max_queue_size_per_topic)) self.retry_queues = defaultdict(lambda: PriorityQueue(maxsize=max_queue_size_per_topic)) self.schema_cache = defaultdict(dict) self.io_loop = IOLoop() self.producer = AsyncProducer(self) self.io_loop.add_callback(self.producer._schedule_retry_periodically) self.producer_thread = Thread(target=self.io_loop.start) self.producer_thread.daemon = True self.producer_thread.start() logger.debug('Started producer background thread') logger.debug('Kafka REST async client initialized for {0}:{1}'.format(self.host, self.port)) def produce(self, topic, value, value_schema, key=None, key_schema=None, partition=None): """Place this message on the appropriate topic queue for asynchronous emission.""" if self.in_shutdown: raise KafkaRESTShutdownException('Client is in shutdown state, new events cannot be produced') if self.schema_cache[topic].get('value') is None: logger.debug('Storing initial value schema for topic {0} in schema cache: {1}'.format(topic, value_schema)) self.schema_cache[topic]['value'] = value_schema if key_schema and self.schema_cache[topic].get('key') is None: logger.debug('Storing initial key schema for topic {0} in schema cache: {1}'.format(topic, key_schema)) self.schema_cache[topic]['key'] = key_schema queue = self.message_queues[topic] message = Message(topic, value, key, partition, 0, 1) try: queue.put_nowait(message) except Full: logger.critical('Primary event queue is full for topic {0}, message {1} will be dropped'.format(topic, message)) self.registrar.emit('drop_message', topic, message, DropReason.PRIMARY_QUEUE_FULL) else: self.registrar.emit('produce', message) self.io_loop.add_callback(self.producer.evaluate_queue, topic, queue) def shutdown(self, block=False): """Prohibit further produce requests and attempt to flush all events currently in the main and retry queues. After this attempt, all remaining events are made available to an event handler but will otherwise be dropped. The producer thread and IOLoop are also shut down. If block=True, this blocks until the producer thread is dead and the shutdown event has been handled.""" logger.info('Client shutting down') self.in_shutdown = True self.io_loop.add_callback(self.producer.start_shutdown) if block: self.producer_thread.join() logger.info('Client completed shutdown') else: logger.info('Client shutting down asynchronously, will not block')
class CompatibilityTests(unittest.TestCase): def setUp(self): self.saved_signals = save_signal_handlers() self.io_loop = IOLoop() self.io_loop.make_current() self.reactor = TornadoReactor(self.io_loop) def tearDown(self): self.reactor.disconnectAll() self.io_loop.clear_current() self.io_loop.close(all_fds=True) restore_signal_handlers(self.saved_signals) def start_twisted_server(self): class HelloResource(Resource): isLeaf = True def render_GET(self, request): return "Hello from twisted!" site = Site(HelloResource()) port = self.reactor.listenTCP(0, site, interface="127.0.0.1") self.twisted_port = port.getHost().port def start_tornado_server(self): class HelloHandler(RequestHandler): def get(self): self.write("Hello from tornado!") app = Application([("/", HelloHandler)], log_function=lambda x: None) server = HTTPServer(app, io_loop=self.io_loop) sock, self.tornado_port = bind_unused_port() server.add_sockets([sock]) def run_ioloop(self): self.stop_loop = self.io_loop.stop self.io_loop.start() self.reactor.fireSystemEvent("shutdown") def run_reactor(self): self.stop_loop = self.reactor.stop self.stop = self.reactor.stop self.reactor.run() def tornado_fetch(self, url, runner): responses = [] client = AsyncHTTPClient(self.io_loop) def callback(response): responses.append(response) self.stop_loop() client.fetch(url, callback=callback) runner() self.assertEqual(len(responses), 1) responses[0].rethrow() return responses[0] def twisted_fetch(self, url, runner): # http://twistedmatrix.com/documents/current/web/howto/client.html chunks = [] client = Agent(self.reactor) d = client.request(b"GET", utf8(url)) class Accumulator(Protocol): def __init__(self, finished): self.finished = finished def dataReceived(self, data): chunks.append(data) def connectionLost(self, reason): self.finished.callback(None) def callback(response): finished = Deferred() response.deliverBody(Accumulator(finished)) return finished d.addCallback(callback) def shutdown(failure): if hasattr(self, "stop_loop"): self.stop_loop() elif failure is not None: # loop hasn't been initialized yet; try our best to # get an error message out. (the runner() interaction # should probably be refactored). try: failure.raiseException() except: logging.error("exception before starting loop", exc_info=True) d.addBoth(shutdown) runner() self.assertTrue(chunks) return "".join(chunks) def twisted_coroutine_fetch(self, url, runner): body = [None] @gen.coroutine def f(): # This is simpler than the non-coroutine version, but it cheats # by reading the body in one blob instead of streaming it with # a Protocol. client = Agent(self.reactor) response = yield client.request(b"GET", utf8(url)) with warnings.catch_warnings(): # readBody has a buggy DeprecationWarning in Twisted 15.0: # https://twistedmatrix.com/trac/changeset/43379 warnings.simplefilter("ignore", category=DeprecationWarning) body[0] = yield readBody(response) self.stop_loop() self.io_loop.add_callback(f) runner() return body[0] def testTwistedServerTornadoClientIOLoop(self): self.start_twisted_server() response = self.tornado_fetch("http://127.0.0.1:%d" % self.twisted_port, self.run_ioloop) self.assertEqual(response.body, "Hello from twisted!") def testTwistedServerTornadoClientReactor(self): self.start_twisted_server() response = self.tornado_fetch("http://127.0.0.1:%d" % self.twisted_port, self.run_reactor) self.assertEqual(response.body, "Hello from twisted!") def testTornadoServerTwistedClientIOLoop(self): self.start_tornado_server() response = self.twisted_fetch("http://127.0.0.1:%d" % self.tornado_port, self.run_ioloop) self.assertEqual(response, "Hello from tornado!") def testTornadoServerTwistedClientReactor(self): self.start_tornado_server() response = self.twisted_fetch("http://127.0.0.1:%d" % self.tornado_port, self.run_reactor) self.assertEqual(response, "Hello from tornado!") @skipIfPy26 def testTornadoServerTwistedCoroutineClientIOLoop(self): self.start_tornado_server() response = self.twisted_coroutine_fetch("http://127.0.0.1:%d" % self.tornado_port, self.run_ioloop) self.assertEqual(response, "Hello from tornado!")
class MockHub(JupyterHub): """Hub with various mock bits""" db_file = None confirm_no_ssl = True last_activity_interval = 2 base_url = '/@/space%20word/' @default('subdomain_host') def _subdomain_host_default(self): return os.environ.get('JUPYTERHUB_TEST_SUBDOMAIN_HOST', '') @default('ip') def _ip_default(self): return '127.0.0.1' @default('authenticator_class') def _authenticator_class_default(self): return MockPAMAuthenticator @default('spawner_class') def _spawner_class_default(self): return MockSpawner def init_signal(self): pass def start(self, argv=None): self.db_file = NamedTemporaryFile() self.pid_file = NamedTemporaryFile(delete=False).name self.db_url = self.db_file.name evt = threading.Event() @gen.coroutine def _start_co(): assert self.io_loop._running # put initialize in start for SQLAlchemy threading reasons yield super(MockHub, self).initialize(argv=argv) # add an initial user user = orm.User(name='user') self.db.add(user) self.db.commit() yield super(MockHub, self).start() yield self.hub.server.wait_up(http=True) self.io_loop.add_callback(evt.set) def _start(): self.io_loop = IOLoop() self.io_loop.make_current() self.io_loop.add_callback(_start_co) self.io_loop.start() self._thread = threading.Thread(target=_start) self._thread.start() ready = evt.wait(timeout=10) assert ready def stop(self): super().stop() self._thread.join() IOLoop().run_sync(self.cleanup) # ignore the call that will fire in atexit self.cleanup = lambda : None self.db_file.close() def login_user(self, name): """Login a user by name, returning her cookies.""" base_url = public_url(self) r = requests.post(base_url + 'hub/login', data={ 'username': name, 'password': name, }, allow_redirects=False, ) r.raise_for_status() assert r.cookies return r.cookies
class Executor(object): """ Distributed executor with data dependencies This executor resembles executors in concurrent.futures but also allows Futures within submit/map calls. Provide center address on initialization >>> executor = Executor(('127.0.0.1', 8787)) # doctest: +SKIP Use ``submit`` method like normal >>> a = executor.submit(add, 1, 2) # doctest: +SKIP >>> b = executor.submit(add, 10, 20) # doctest: +SKIP Additionally, provide results of submit calls (futures) to further submit calls: >>> c = executor.submit(add, a, b) # doctest: +SKIP This allows for the dynamic creation of complex dependencies. """ def __init__(self, center, start=True, delete_batch_time=1): self.center = coerce_to_rpc(center) self.futures = dict() self.refcount = defaultdict(lambda: 0) self.dask = dict() self.restrictions = dict() self.loop = IOLoop() self.report_queue = Queue() self.scheduler_queue = Queue() self._shutdown_event = Event() self._delete_batch_time = delete_batch_time if start: self.start() def start(self): """ Start scheduler running in separate thread """ from threading import Thread self.loop.add_callback(self._go) self._loop_thread = Thread(target=self.loop.start) self._loop_thread.start() def __enter__(self): if not self.loop._running: self.start() return self def __exit__(self, type, value, traceback): self.shutdown() def _inc_ref(self, key): self.refcount[key] += 1 def _dec_ref(self, key): self.refcount[key] -= 1 if self.refcount[key] == 0: del self.refcount[key] self._release_key(key) def _release_key(self, key): """ Release key from distributed memory """ self.futures[key]['event'].clear() logger.debug("Release key %s", key) del self.futures[key] self.scheduler_queue.put_nowait({'op': 'release-held-data', 'key': key}) @gen.coroutine def report(self): """ Listen to scheduler """ while True: msg = yield self.report_queue.get() if msg['op'] == 'close': break if msg['op'] == 'task-finished': if msg['key'] in self.futures: self.futures[msg['key']]['status'] = 'finished' self.futures[msg['key']]['event'].set() if msg['op'] == 'lost-data': if msg['key'] in self.futures: self.futures[msg['key']]['status'] = 'lost' self.futures[msg['key']]['event'].clear() if msg['op'] == 'task-erred': if msg['key'] in self.futures: self.futures[msg['key']]['status'] = 'error' self.futures[msg['key']]['event'].set() @gen.coroutine def _shutdown(self): """ Send shutdown signal and wait until _go completes """ self.report_queue.put_nowait({'op': 'close'}) self.scheduler_queue.put_nowait({'op': 'close'}) yield self._shutdown_event.wait() def shutdown(self): """ Send shutdown signal and wait until scheduler terminates """ self.report_queue.put_nowait({'op': 'close'}) self.scheduler_queue.put_nowait({'op': 'close'}) self.loop.stop() self._loop_thread.join() @gen.coroutine def _go(self): """ Setup and run all other coroutines. Block until finished. """ self.who_has, self.has_what, self.ncores = yield [self.center.who_has(), self.center.has_what(), self.center.ncores()] self.waiting = {} self.processing = {} self.stacks = {} worker_queues = {worker: Queue() for worker in self.ncores} delete_queue = Queue() coroutines = ([ self.report(), scheduler(self.scheduler_queue, self.report_queue, worker_queues, delete_queue, self.who_has, self.has_what, self.ncores, self.dask, self.restrictions, self.waiting, self.stacks, self.processing), delete(self.scheduler_queue, delete_queue, self.center.ip, self.center.port, self._delete_batch_time)] + [worker(self.scheduler_queue, worker_queues[w], w, n) for w, n in self.ncores.items()]) results = yield All(coroutines) self._shutdown_event.set() def submit(self, func, *args, **kwargs): """ Submit a function application to the scheduler Parameters ---------- func: callable *args: **kwargs: pure: bool (defaults to True) Whether or not the function is pure. Set ``pure=False`` for impure functions like ``np.random.random``. workers: set, iterable of sets A set of worker hostnames on which computations may be performed. Leave empty to default to all workers (common case) Examples -------- >>> c = executor.submit(add, a, b) # doctest: +SKIP Returns ------- Future See Also -------- distributed.executor.Executor.submit: """ if not callable(func): raise TypeError("First input to submit must be a callable function") key = kwargs.pop('key', None) pure = kwargs.pop('pure', True) workers = kwargs.pop('workers', None) if key is None: if pure: key = funcname(func) + '-' + tokenize(func, kwargs, *args) else: key = funcname(func) + '-' + next(tokens) if key in self.futures: return Future(key, self) if kwargs: task = (apply, func, args, kwargs) else: task = (func,) + args if workers is not None: restrictions = {key: workers} else: restrictions = {} if key not in self.futures: self.futures[key] = {'event': Event(), 'status': 'waiting'} logger.debug("Submit %s(...), %s", funcname(func), key) self.scheduler_queue.put_nowait({'op': 'update-graph', 'dsk': {key: task}, 'keys': [key], 'restrictions': restrictions}) return Future(key, self) def map(self, func, *iterables, **kwargs): """ Map a function on a sequence of arguments Arguments can be normal objects or Futures Parameters ---------- func: callable iterables: Iterables pure: bool (defaults to True) Whether or not the function is pure. Set ``pure=False`` for impure functions like ``np.random.random``. workers: set, iterable of sets A set of worker hostnames on which computations may be performed. Leave empty to default to all workers (common case) Examples -------- >>> L = executor.map(func, sequence) # doctest: +SKIP Returns ------- list of futures See also -------- distributed.executor.Executor.submit """ pure = kwargs.pop('pure', True) workers = kwargs.pop('workers', None) if not callable(func): raise TypeError("First input to map must be a callable function") iterables = [list(it) for it in iterables] if pure: keys = [funcname(func) + '-' + tokenize(func, kwargs, *args) for args in zip(*iterables)] else: uid = str(uuid.uuid4()) keys = [funcname(func) + '-' + uid + '-' + next(tokens) for i in range(min(map(len, iterables)))] if not kwargs: dsk = {key: (func,) + args for key, args in zip(keys, zip(*iterables))} else: dsk = {key: (apply, func, args, kwargs) for key, args in zip(keys, zip(*iterables))} for key in dsk: if key not in self.futures: self.futures[key] = {'event': Event(), 'status': 'waiting'} if isinstance(workers, (list, set)): if workers and isinstance(first(workers), (list, set)): if len(workers) != len(keys): raise ValueError("You only provided %d worker restrictions" " for a sequence of length %d" % (len(workers), len(keys))) restrictions = dict(zip(keys, workers)) else: restrictions = {key: workers for key in keys} elif workers is None: restrictions = {} else: raise TypeError("Workers must be a list or set of workers or None") logger.debug("map(%s, ...)", funcname(func)) self.scheduler_queue.put_nowait({'op': 'update-graph', 'dsk': dsk, 'keys': keys, 'restrictions': restrictions}) return [Future(key, self) for key in keys] @gen.coroutine def _gather(self, futures): futures2, keys = unpack_remotedata(futures) keys = list(keys) while True: yield All([self.futures[key]['event'].wait() for key in keys]) try: data = yield _gather(self.center, keys) except KeyError as e: self.scheduler_queue.put_nowait({'op': 'missing-data', 'missing': e.args}) for key in e.args: self.futures[key]['event'].clear() else: break data = dict(zip(keys, data)) result = pack_data(futures2, data) raise gen.Return(result) def gather(self, futures): """ Gather futures from distributed memory Accepts a future or any nested core container of futures Examples -------- >>> from operator import add # doctest: +SKIP >>> e = Executor('127.0.0.1:8787') # doctest: +SKIP >>> x = e.submit(add, 1, 2) # doctest: +SKIP >>> e.gather(x) # doctest: +SKIP 3 >>> e.gather([x, [x], x]) # doctest: +SKIP [3, [3], 3] """ return sync(self.loop, self._gather, futures) @gen.coroutine def _get(self, dsk, keys, restrictions=None): flatkeys = list(flatten(keys)) for key in flatkeys: if key not in self.futures: self.futures[key] = {'event': Event(), 'status': None} futures = {key: Future(key, self) for key in flatkeys} self.scheduler_queue.put_nowait({'op': 'update-graph', 'dsk': dsk, 'keys': flatkeys, 'restrictions': restrictions or {}}) packed = pack_data(keys, futures) result = yield self._gather(packed) raise gen.Return(result) def get(self, dsk, keys, **kwargs): """ Gather futures from distributed memory Parameters ---------- dsk: dict keys: object, or nested lists of objects restrictions: dict (optional) A mapping of {key: {set of worker hostnames}} that restricts where jobs can take place Examples -------- >>> from operator import add # doctest: +SKIP >>> e = Executor('127.0.0.1:8787') # doctest: +SKIP >>> e.get({'x': (add, 1, 2)}, 'x') # doctest: +SKIP 3 """ return sync(self.loop, self._get, dsk, keys, **kwargs)