def test__yield_for_all_futures(): loop = IOLoop() loop.make_current() @gen.coroutine def several_steps(): value = 0 value += yield async_value(1) value += yield async_value(2) value += yield async_value(3) raise gen.Return(value) result = {} def on_done(future): result["value"] = future.result() loop.stop() loop.add_future(yield_for_all_futures(several_steps()), on_done) try: loop.start() except KeyboardInterrupt as e: print("keyboard interrupt") assert 6 == result["value"] loop.close()
def test_flexx_in_thread4(): """ Test threading starting server in other thread where it is created. """ res = [] loop = IOLoop() loop.make_current() app.create_server() def try_start(): try: app.stop() app.start() except RuntimeError: res.append('start-fail') else: res.append('start-ok') def main(): app.create_server() try_start() # Try to start server that was created in other thread -> fail t = threading.Thread(target=try_start) t.start() t.join() # Try to start in same thread as created -> ok t = threading.Thread(target=main) t.start() t.join() assert res == ['start-fail', 'start-ok']
def test_rebinding_ioloop(): """ Test recreating server objects, and its binding to the current ioloop. """ res = [] def add_res(i): res.append(i) # Create new ioloop loop = IOLoop() loop.make_current() # Create new flexx server, which binds to that loop server1 = app.create_server() assert server1 is app.current_server() # assert loop is server1._loop # Create new ioloop loop = IOLoop() loop.make_current() # This is a new loop assert loop is not server1._loop # Create new flexx server, which binds to that loop server2 = app.create_server() assert server2 is app.current_server() assert server1 is not server2 # assert loop is server2._loop
def test_flexx_in_thread1(): """ Test threading and ioloop selection. """ def main(): loop2.make_current() app.create_server() # Create 3 loops, nr 2 is made current in the thread loop1 = IOLoop() loop2 = IOLoop() loop3 = IOLoop() loop1.make_current() server1 = app.create_server() t = threading.Thread(target=main) t.start() t.join() server2 = app.current_server() # still current as set by the thread loop3.make_current() server3 = app.create_server() assert server1._loop is loop1 assert server2._loop is loop2 assert server3._loop is loop3
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port, worker_port, local_dir, services, name, memory_limit): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name, memory_limit=memory_limit, loop=loop) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover loop.add_callback(start) # pragma: no cover try: loop.start() # pragma: no cover finally: loop.stop() loop.close(all_fds=True)
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port, local_dir, services): """ Function run by the Nanny when creating the worker """ from distributed import Worker from tornado.ioloop import IOLoop IOLoop.clear_instance() loop = IOLoop() loop.make_current() worker = Worker(center_ip, center_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services) @gen.coroutine def start(): try: yield worker._start() except Exception as e: logger.exception(e) q.put(e) else: assert worker.port q.put({'port': worker.port, 'dir': worker.local_dir}) loop.add_callback(start) loop.start()
def loop(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() yield loop loop.stop() loop.close()
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port, worker_port, local_dir, services, name): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(center_ip, center_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover loop.add_callback(start) # pragma: no cover with ignoring(KeyboardInterrupt): loop.start() # pragma: no cover
def test_instance_pool(self): instance1 = _Instance() instance2 = _Instance() pool = InstancePool([instance1, instance2]) pool.increment() pool.increment() self.assertEquals(instance1.value(), 1) self.assertEquals(instance2.value(), 1) pool.transaction(lambda i: i.increment()) pool.transaction(lambda i: i.increment()) self.assertEquals(instance1.value(), 2) self.assertEquals(instance2.value(), 2) @coroutine def yield_tasks(): self.assertEquals((yield pool.await().increment()), 3) self.assertEquals((yield pool.await().increment()), 3) self.assertEquals(instance1.value(), 3) self.assertEquals(instance2.value(), 3) self.assertEquals((yield pool.await_transaction(lambda i: i.increment())), 4) self.assertEquals((yield pool.await_transaction(lambda i: i.increment())), 4) loop = IOLoop() loop.make_current() loop.run_sync(yield_tasks) self.assertEquals(instance1.value(), 4) self.assertEquals(instance2.value(), 4)
class XDebugServer(TCPServer): """Class to listen for xdebug requests""" def __init__(self): """Constructor """ self.ioloop = IOLoop() super(XDebugServer, self).__init__(io_loop=self.ioloop) self.listen(9000) # this is for cross thread communication self.inport = Queue() self.outport = Queue() self._xdebug_connection = None def listenfunc(): self.ioloop.make_current() self.ioloop.start() self.ioloop.close(all_fds=True) self.listener_thread = threading.Thread(target=listenfunc) self.listener_thread.daemon = True self.listener_thread.start() def handle_stream(self, stream, address): """Handle a connection Only one connection at a time, for now :stream: @todo :address: @todo :returns: @todo """ self._xdebug_connection = XDebugConnection(self, stream, address) def run_command(self, command, data=None): """Send status :returns: @todo """ self.inport.put("{} -i 1\0".format(str(command))) return self.outport.get() def stop(self): """Stop tornado event loop :returns: @todo """ self.ioloop.stop() self.listener_thread.join() del self.ioloop del self.listener_thread
def pristine_loop(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() try: yield loop finally: loop.close(all_fds=True) IOLoop.clear_instance()
def _run(): io_loop = IOLoop() io_loop.make_current() io_loop.add_callback(lambda : evt.set()) with mock.patch.dict(os.environ, env): app = self._app = MockSingleUserServer() app.initialize(args) app.start()
def run_server(port, daemon='start'): application = tornado.web.Application([ (r"/", TestHandler, {'port': port}), ]) ioloop = IOLoop() ioloop.make_current() SERVER_LOOPS.append(ioloop) application.listen(port) ioloop.start()
def atexit(self): """atexit callback""" if self._atexit_ran: return self._atexit_ran = True # run the cleanup step (in a new loop, because the interrupted one is unclean) IOLoop.clear_current() loop = IOLoop() loop.make_current() loop.run_sync(self.cleanup)
def loop(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() yield loop sync(loop, loop.stop) for i in range(5): with ignoring(Exception): loop.close(all_fds=True) break
def _atexit(self): if self._atexit_ran: return self._atexit_ran = True self._stats_job.stop() IOLoop.clear_current() loop = IOLoop() loop.make_current() loop.run_sync(self._cleanup)
def run_center(port): from distributed import Center from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = Center('127.0.0.1', port) center.listen(port) loop.start()
def test_func(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() cor = gen.coroutine(func) try: loop.run_sync(cor, timeout=timeout) finally: loop.stop() loop.close(all_fds=True)
def run_nanny(port, center_port, **kwargs): from distributed import Nanny from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Nanny('127.0.0.1', port, port + 1000, '127.0.0.1', center_port, **kwargs) loop.run_sync(worker._start) loop.start()
def _run(): asyncio.set_event_loop(asyncio.new_event_loop()) io_loop = IOLoop() io_loop.make_current() io_loop.add_callback(lambda: evt.set()) with mock.patch.dict(os.environ, env): app = self._app = MockSingleUserServer() app.initialize(args) assert app.hub_auth.oauth_client_id assert app.hub_auth.api_token app.start()
def test_func(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() s, workers = loop.run_sync(lambda: start_cluster(ncores)) try: loop.run_sync(lambda: cor(s, *workers), timeout=timeout) finally: loop.run_sync(lambda: end_cluster(s, workers)) loop.stop() loop.close()
def main(): # Create fresh ioloop and make flexx use it loop = IOLoop() loop.make_current() app.create_server() # Create model and manipulate prop model = MyModel1() model.foo = 3 model.foo = 4 # Run mainloop for one iterartion app.call_later(0, app.stop) app.start()
def run_worker(port, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop() loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker("127.0.0.1", port, "127.0.0.1", center_port, **kwargs) worker.start() loop.start()
def run_worker(q, center_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback import logging with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) loop.start()
def _initialize(queue): result = None try: # create new IOLoop in the thread io_loop = IOLoop() # make it default for that thread io_loop.make_current() result = io_loop io_loop.add_callback(queue.put, result) io_loop.start() except Exception as err: # pragma: no cover result = err finally: # pragma: no cover queue.put(result)
def pristine_loop(): IOLoop.clear_instance() IOLoop.clear_current() loop = IOLoop() loop.make_current() assert IOLoop.current() is loop try: yield loop finally: try: loop.close(all_fds=True) except (KeyError, ValueError): pass IOLoop.clear_instance() IOLoop.clear_current()
def run_scheduler(q, scheduler_port=0, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() scheduler = Scheduler(loop=loop, validate=True, **kwargs) done = scheduler.start(scheduler_port) q.put(scheduler.port) try: loop.start() finally: loop.close(all_fds=True)
def run_worker(q, scheduler_port, **kwargs): from distributed import Worker from tornado.ioloop import IOLoop, PeriodicCallback with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1', loop=loop, validate=True, **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.close(all_fds=True)
def current_loop(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() yield loop if loop._running: sync(loop, loop.stop) for i in range(5): try: loop.close(all_fds=True) return except Exception as e: f = e print(f) IOLoop.clear_instance()
def test_awaitable(self): instance = _Instance() instance.increment() self.assertEquals(instance.value(), 1) awaitable = AwaitableInstance(instance) @coroutine def yield_tasks(): self.assertEquals((yield awaitable.increment()), 2) self.assertEquals((yield awaitable.increment()), 3) self.assertEquals((yield awaitable.increment()), 4) self.assertEquals((yield awaitable.value()), 4) loop = IOLoop() loop.make_current() loop.run_sync(yield_tasks) self.assertEquals(instance.value(), 4)
class LoopAndGroup: def __init__(self, quit_after=None) -> None: self.io_loop = IOLoop() self.io_loop.make_current() self.group = _CallbackGroup(self.io_loop) if quit_after is not None: self.io_loop.call_later(quit_after / 1000.0, lambda: self.io_loop.stop()) def __exit__(self, type, value, traceback): run(self.io_loop) self.io_loop.close() def __enter__(self): return self
def run_nanny(q, center_port, **kwargs): from distributed import Nanny from tornado.ioloop import IOLoop, PeriodicCallback import logging with log_errors(): IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) worker = Nanny('127.0.0.1', center_port, ip='127.0.0.1', **kwargs) loop.run_sync(lambda: worker._start(0)) q.put(worker.port) try: loop.start() finally: loop.close(all_fds=True)
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port, worker_port, local_dir, services, name, memory_limit, reconnect, resources, validate): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name, memory_limit=memory_limit, reconnect=reconnect, validate=validate, resources=resources, loop=loop) # pragma: no cover @gen.coroutine # pragma: no cover def run(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({ 'port': worker.port, 'dir': worker.local_dir }) # pragma: no cover while worker.status != 'closed': yield gen.sleep(0.1) logger.info("Worker closed") try: loop.run_sync(run) finally: loop.stop() loop.close(all_fds=True)
def test_base_server() -> None: app = BokehTornado(Application()) httpserver = HTTPServer(app) httpserver.start() loop = IOLoop() loop.make_current() server = BaseServer(loop, app, httpserver) server.start() assert server.io_loop == loop assert server._tornado.io_loop == loop httpserver.stop() server.stop() server.io_loop.close()
def prepare_browser(project, stage, io_loop, show_url): from tornado.ioloop import IOLoop from conda_kapsel.internal.ui_server import UIServer, UIServerDoneEvent result_holder = {} old_current_loop = None try: old_current_loop = IOLoop.current() if io_loop is None: io_loop = IOLoop() io_loop.make_current() if show_url is None: show_url = _default_show_url def event_handler(event): if isinstance(event, UIServerDoneEvent): result_holder['result'] = event.result io_loop.stop() server = UIServer(project, stage, event_handler=event_handler, io_loop=io_loop) try: print("# Configure the project at {url} to continue...".format( url=server.url)) show_url(server.url) io_loop.start() finally: server.unlisten() finally: if old_current_loop is not None: old_current_loop.make_current() if 'result' in result_holder: return result_holder['result'] else: from conda_kapsel.prepare import PrepareFailure # this pretty much only happens in unit tests. return PrepareFailure(logs=[], statuses=(), errors=["Browser UI main loop was stopped."], environ=stage.environ, overrides=stage.overrides)
def do_test(dirname): io_loop = IOLoop() io_loop.make_current() events = [] def event_handler(event): events.append(event) project = Project(dirname) local_state_file = LocalStateFile.load_for_directory(dirname) requirement = EnvVarRequirement(registry=project.plugin_registry, env_var="FOO") status = requirement.check_status(dict(), local_state_file, 'default', UserConfigOverrides()) context = ConfigurePrepareContext(dict(), local_state_file, 'default', UserConfigOverrides(), [status]) server = UIServer(project, _no_op_prepare(context), event_handler, io_loop) # do a get so that _requirements_by_id below exists get_response = http_get(io_loop, server.url) assert 200 == get_response.code req_id = list(server._application._requirements_by_id.keys())[0] if '%s' in name_template: name = name_template % req_id else: name = name_template encoder = MultipartEncoder({name: 'bloop'}) body = encoder.to_string() headers = {'Content-Type': encoder.content_type} post_response = http_post(io_loop, server.url, body=body, headers=headers) # we just ignore bad form names, because they are assumed # to be some sort of hostile thing. we shouldn't ever # generate them on purpose. assert 200 == post_response.code server.unlisten() assert len(events) == 1 assert isinstance(events[0], UIServerDoneEvent) out, err = capsys.readouterr() assert out == "" assert err == expected_err
class LoopOverhead(object): """ These are not distributed benchmarks per se, but help assessing Tornado's loop management overhead for other benchmarks. """ def setup(self): self.loop = IOLoop() self.loop.make_current() def time_loop_start_stop(self): self.loop.add_callback(self.loop.stop) self.loop.start() @gen.coroutine def _empty_coro(self): pass def time_loop_run_sync(self): run_sync(self.loop, self._empty_coro)
def run_scheduler(q, center_port=None, **kwargs): from distributed import Scheduler from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = ('127.0.0.1', center_port) if center_port else None scheduler = Scheduler(center=center, **kwargs) scheduler.listen(0) if center_port: loop.run_sync(scheduler.sync_center) done = scheduler.start(0) q.put(scheduler.port) loop.start()
def test_restarting(): """ Test stopping and starting the ioloop. """ res = [] def add_res(i): res.append(i) def try_start(): try: app.start() except RuntimeError: res.append('RTE') # Create new ioloop always loop = IOLoop() loop.make_current() # Make Flexx use it server = app.create_server() app.call_later(0, add_res, 1) app.call_later(0, add_res, 2) app.call_later( 0, app.stop) # actually, just calling stop() would work as well app.start() assert server._running == False app.call_later(0, try_start) # test that cannot start twice app.call_later(0, add_res, 3) app.call_later(0, add_res, 4) app.call_later(0, app.stop) app.start() assert server._running == False app.call_later(0, try_start) # test that cannot start twice app.call_later(0, add_res, 5) app.call_later(0, add_res, 6) app.call_later(0, app.stop) app.start() assert server._running == False assert res == [1, 2, 'RTE', 3, 4, 'RTE', 5, 6]
def do_test(dirname): io_loop = IOLoop() io_loop.make_current() events = [] def event_handler(event): events.append(event) local_state_file = LocalStateFile.load_for_directory(dirname) value = local_state_file.get_value(['variables', 'FOO']) assert value is None project = Project(dirname) requirement = EnvVarRequirement(registry=project.plugin_registry, env_var="FOO") status = requirement.check_status(dict(), local_state_file, 'default', UserConfigOverrides()) context = ConfigurePrepareContext(dict(), local_state_file, 'default', UserConfigOverrides(), [status]) server = UIServer(project, _no_op_prepare(context), event_handler, io_loop) get_response = http_get(io_loop, server.url) print(repr(get_response)) soup = BeautifulSoup(get_response.body, _BEAUTIFUL_SOUP_BACKEND) field = soup.find_all("input", attrs={'type': 'text'})[0] assert 'name' in field.attrs encoder = MultipartEncoder({field['name']: 'bloop'}) body = encoder.to_string() headers = {'Content-Type': encoder.content_type} post_response = http_post(io_loop, server.url, body=body, headers=headers) print(repr(post_response)) server.unlisten() assert len(events) == 1 assert isinstance(events[0], UIServerDoneEvent) value = local_state_file.get_value(['variables', 'FOO']) assert 'bloop' == value
def pristine_loop(): """ Builds a clean IOLoop for using as a background request. Courtesy of Dask Distributed """ IOLoop.clear_instance() IOLoop.clear_current() loop = IOLoop() loop.make_current() assert IOLoop.current() is loop try: yield loop finally: try: loop.close(all_fds=True) except (ValueError, KeyError, RuntimeError): pass IOLoop.clear_instance() IOLoop.clear_current()
def run_center(q): from distributed import Center from tornado.ioloop import IOLoop, PeriodicCallback import logging IOLoop.clear_instance() loop = IOLoop(); loop.make_current() PeriodicCallback(lambda: None, 500).start() logging.getLogger("tornado").setLevel(logging.CRITICAL) center = Center('127.0.0.1') while True: try: center.listen(0) break except Exception as e: logging.info("Could not start center on port. Retrying", exc_info=True) q.put(center.port) loop.start()
class ControlThread(Thread): def __init__(self, **kwargs): Thread.__init__(self, **kwargs) self.io_loop = IOLoop(make_current=False) self.pydev_do_not_trace = True self.is_pydev_daemon_thread = True def run(self): self.io_loop.make_current() try: self.io_loop.start() finally: self.io_loop.close() def stop(self): """Stop the thread. This method is threadsafe. """ self.io_loop.add_callback(self.io_loop.stop)
def run_worker_fork(q, scheduler_addr, ncores, nanny_port, worker_ip, worker_port, local_dir, **kwargs): """ Create a worker by forking. """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(scheduler_addr, ncores=ncores, service_ports={'nanny': nanny_port}, local_dir=local_dir, **kwargs) # pragma: no cover @gen.coroutine # pragma: no cover def run(): try: # pragma: no cover yield worker._start((worker_ip, worker_port)) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({ 'address': worker.address, 'dir': worker.local_dir }) # pragma: no cover yield worker.wait_until_closed() logger.info("Worker closed") try: loop.run_sync(run) except TimeoutError: logger.info("Worker timed out") finally: loop.stop() loop.close(all_fds=True)
def test_more_stopping(): """ Test calling stop multiple times. """ # This is why you want to create new IOLoop instances for each test # Create new ioloop and make Flexx use it loop = IOLoop() loop.make_current() server = app.create_server() app.stop() # triggers event to stop app.start() app.stop() # Extra stop - pending stop event # Which means the next stop does hardly block t0 = time.time() app.call_later(0.2, app.stop) app.start() assert time.time() - t0 < 0.1 loop = IOLoop() loop.make_current() server = app.create_server() # But stops dont stack app.stop() app.stop() app.stop() app.stop() # Flush all stops ... app.stop() app.start() # ... so that we now have an expected loop t0 = time.time() app.call_later(0.2, app.stop) app.start() assert time.time() - t0 >= 0.1
def test_func(): IOLoop.clear_instance() loop = IOLoop() loop.make_current() s, workers = loop.run_sync( lambda: start_cluster(ncores, loop, Worker=Worker)) args = [s] + workers if client: e = Client((s.ip, s.port), loop=loop, start=False) loop.run_sync(e._start) args = [e] + args try: loop.run_sync(lambda: cor(*args), timeout=timeout) finally: if client: loop.run_sync(e._shutdown) loop.run_sync(lambda: end_cluster(s, workers)) loop.stop() loop.close(all_fds=True)
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port, worker_port, local_dir, services, name, memory_limit): """ Function run by the Nanny when creating the worker """ from distributed import Worker # pragma: no cover from tornado.ioloop import IOLoop # pragma: no cover IOLoop.clear_instance() # pragma: no cover loop = IOLoop() # pragma: no cover loop.make_current() # pragma: no cover worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip, service_ports={'nanny': nanny_port}, local_dir=local_dir, services=services, name=name, memory_limit=memory_limit, loop=loop) # pragma: no cover @gen.coroutine # pragma: no cover def start(): try: # pragma: no cover yield worker._start(worker_port) # pragma: no cover except Exception as e: # pragma: no cover logger.exception(e) # pragma: no cover q.put(e) # pragma: no cover else: assert worker.port # pragma: no cover q.put({ 'port': worker.port, 'dir': worker.local_dir }) # pragma: no cover loop.add_callback(start) # pragma: no cover try: loop.start() # pragma: no cover finally: loop.stop() loop.close(all_fds=True)
def do_test(dirname): io_loop = IOLoop() io_loop.make_current() events = [] def event_handler(event): events.append(event) project = Project(dirname) local_state_file = LocalStateFile.load_for_directory(dirname) context = ConfigurePrepareContext(dict(), local_state_file, 'default', UserConfigOverrides(), []) server = UIServer(project, _no_op_prepare(context), event_handler, io_loop) get_response = http_get(io_loop, server.url) print(repr(get_response)) post_response = http_post(io_loop, server.url, body="") print(repr(post_response)) server.unlisten() assert len(events) == 1 assert isinstance(events[0], UIServerDoneEvent)
def test__ioloop_not_forcibly_stopped(): # Issue #5494 application = Application() loop = IOLoop() loop.make_current() server = Server(application, ioloop=loop) server.start() result = [] def f(): server.unlisten() server.stop() # If server.stop() were to stop the Tornado IO loop, # g() wouldn't be called and `result` would remain empty. loop.add_timeout(timedelta(seconds=0.01), g) def g(): result.append(None) loop.stop() loop.add_callback(f) loop.start() assert result == [None]
class Connect(object): """ Test overhead of connect() and Comm.close(). """ N_CONNECTS = 100 def setup(self): self.loop = IOLoop() self.loop.make_current() @gen.coroutine def _handle_comm(self, comm): yield comm.close() @gen.coroutine def _connect_close(self, addr): comm = yield connect(addr) yield comm.close() @gen.coroutine def _main(self, address): listener = listen(address, self._handle_comm) listener.start() yield [ self._connect_close(listener.contact_address) for i in range(self.N_CONNECTS) ] listener.stop() def _time_connect(self, address): run_sync(self.loop, partial(self._main, address)) def time_tcp_connect(self): self._time_connect('tcp://127.0.0.1') def time_inproc_connect(self): self._time_connect('inproc://')
def run(self, loop=None): ''' Start servicing the Tornado event loop. ''' if not loop: loop = IOLoop() loop.make_current() # bind the socket self.listen(self._port, self._address) logger.info('Pensive started on {}:{}'.format(self._address or '*', self._port)) try: loop.start() except KeyboardInterrupt: pass loop.stop() loop.close() logger.info('Pensive stopped')
def main(): """ Main function for this module. Load and run tests. Each test run a game and checks game messages and phases against an expected game data file. Current tested gama data files are JSON files located into folder diplomacy/tests/network. """ parser = argparse.ArgumentParser(description='Run test cases against an external server to connect.') parser.add_argument('--port', type=int, required=True, help='run on the given port (required)') parser.add_argument('--cases', action='append', help="Run given cases. " "Each case <C> must match a test case file <C>.json located in diplomacy.tests.network. " "If not provided, all available cases are run.") args = parser.parse_args() io_loop = IOLoop() io_loop.make_current() @gen.coroutine def run(): """ Run all tests consecutively in one call. """ tests = set(args.cases) if args.cases else {'1', '2', '3'} for test_case in list(sorted(tests)): yield launch_case('%s.json' % test_case, args.port, io_loop) io_loop.run_sync(run)
def __init__(self, application, **server_kwargs): loop = IOLoop() loop.make_current() server_kwargs['io_loop'] = loop self._server = Server(application, **server_kwargs)
def _run( cls, worker_kwargs, worker_start_args, silence_logs, init_result_q, child_stop_q, uid, env, config, Worker, ): # pragma: no cover os.environ.update(env) dask.config.set(config) try: from dask.multiprocessing import initialize_worker_process except ImportError: # old Dask version pass else: initialize_worker_process() if silence_logs: logger.setLevel(silence_logs) IOLoop.clear_instance() loop = IOLoop() loop.make_current() worker = Worker(**worker_kwargs) async def do_stop(timeout=5, executor_wait=True): try: await worker.close( report=False, nanny=False, executor_wait=executor_wait, timeout=timeout, ) finally: loop.stop() def watch_stop_q(): """ Wait for an incoming stop message and then stop the worker cleanly. """ while True: try: msg = child_stop_q.get(timeout=1000) except Empty: pass else: child_stop_q.close() assert msg.pop("op") == "stop" loop.add_callback(do_stop, **msg) break t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch") t.daemon = True t.start() async def run(): """ Try to start worker and inform parent of outcome. """ try: await worker except Exception as e: logger.exception("Failed to start worker") init_result_q.put({"uid": uid, "exception": e}) init_result_q.close() else: try: assert worker.address except ValueError: pass else: init_result_q.put( { "address": worker.address, "dir": worker.local_directory, "uid": uid, } ) init_result_q.close() await worker.finished() logger.info("Worker closed") try: loop.run_sync(run) except TimeoutError: # Loop was stopped before wait_until_closed() returned, ignore pass except KeyboardInterrupt: pass
class CompatibilityTests(unittest.TestCase): def setUp(self): self.saved_signals = save_signal_handlers() self.io_loop = IOLoop() self.io_loop.make_current() self.reactor = AsyncioSelectorReactor() def tearDown(self): self.reactor.disconnectAll() self.io_loop.clear_current() self.io_loop.close(all_fds=True) restore_signal_handlers(self.saved_signals) def start_twisted_server(self): class HelloResource(Resource): isLeaf = True def render_GET(self, request): return b"Hello from twisted!" site = Site(HelloResource()) port = self.reactor.listenTCP(0, site, interface='127.0.0.1') self.twisted_port = port.getHost().port def start_tornado_server(self): class HelloHandler(RequestHandler): def get(self): self.write("Hello from tornado!") app = Application([('/', HelloHandler)], log_function=lambda x: None) server = HTTPServer(app) sock, self.tornado_port = bind_unused_port() server.add_sockets([sock]) def run_reactor(self): # In theory, we can run the event loop through Tornado, # Twisted, or asyncio interfaces. However, since we're trying # to avoid installing anything as the global event loop, only # the twisted interface gets everything wired up correectly # without extra hacks. This method is a part of a # no-longer-used generalization that allowed us to test # different combinations. self.stop_loop = self.reactor.stop self.stop = self.reactor.stop self.reactor.run() def tornado_fetch(self, url, runner): client = AsyncHTTPClient() fut = client.fetch(url) fut.add_done_callback(lambda f: self.stop_loop()) runner() return fut.result() def twisted_fetch(self, url, runner): # http://twistedmatrix.com/documents/current/web/howto/client.html chunks = [] client = Agent(self.reactor) d = client.request(b'GET', utf8(url)) class Accumulator(Protocol): def __init__(self, finished): self.finished = finished def dataReceived(self, data): chunks.append(data) def connectionLost(self, reason): self.finished.callback(None) def callback(response): finished = Deferred() response.deliverBody(Accumulator(finished)) return finished d.addCallback(callback) def shutdown(failure): if hasattr(self, 'stop_loop'): self.stop_loop() elif failure is not None: # loop hasn't been initialized yet; try our best to # get an error message out. (the runner() interaction # should probably be refactored). try: failure.raiseException() except: logging.error('exception before starting loop', exc_info=True) d.addBoth(shutdown) runner() self.assertTrue(chunks) return b''.join(chunks) def twisted_coroutine_fetch(self, url, runner): body = [None] @gen.coroutine def f(): # This is simpler than the non-coroutine version, but it cheats # by reading the body in one blob instead of streaming it with # a Protocol. client = Agent(self.reactor) response = yield client.request(b'GET', utf8(url)) with warnings.catch_warnings(): # readBody has a buggy DeprecationWarning in Twisted 15.0: # https://twistedmatrix.com/trac/changeset/43379 warnings.simplefilter('ignore', category=DeprecationWarning) body[0] = yield readBody(response) self.stop_loop() self.io_loop.add_callback(f) runner() return body[0] def testTwistedServerTornadoClientReactor(self): self.start_twisted_server() response = self.tornado_fetch( 'http://127.0.0.1:%d' % self.twisted_port, self.run_reactor) self.assertEqual(response.body, b'Hello from twisted!') def testTornadoServerTwistedClientReactor(self): self.start_tornado_server() response = self.twisted_fetch( 'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor) self.assertEqual(response, b'Hello from tornado!') def testTornadoServerTwistedCoroutineClientReactor(self): self.start_tornado_server() response = self.twisted_coroutine_fetch( 'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor) self.assertEqual(response, b'Hello from tornado!')
class MockHub(JupyterHub): """Hub with various mock bits""" db_file = None confirm_no_ssl = True last_activity_interval = 2 @default('subdomain_host') def _subdomain_host_default(self): return os.environ.get('JUPYTERHUB_TEST_SUBDOMAIN_HOST', '') @default('ip') def _ip_default(self): return '127.0.0.1' @default('authenticator_class') def _authenticator_class_default(self): return MockPAMAuthenticator @default('spawner_class') def _spawner_class_default(self): return MockSpawner def init_signal(self): pass def start(self, argv=None): self.db_file = NamedTemporaryFile() self.pid_file = NamedTemporaryFile(delete=False).name self.db_url = self.db_file.name evt = threading.Event() @gen.coroutine def _start_co(): assert self.io_loop._running # put initialize in start for SQLAlchemy threading reasons yield super(MockHub, self).initialize(argv=argv) # add an initial user user = orm.User(name='user') self.db.add(user) self.db.commit() yield super(MockHub, self).start() yield self.hub.server.wait_up(http=True) self.io_loop.add_callback(evt.set) def _start(): self.io_loop = IOLoop() self.io_loop.make_current() self.io_loop.add_callback(_start_co) self.io_loop.start() self._thread = threading.Thread(target=_start) self._thread.start() ready = evt.wait(timeout=10) assert ready def stop(self): super().stop() self._thread.join() IOLoop().run_sync(self.cleanup) # ignore the call that will fire in atexit self.cleanup = lambda: None self.db_file.close() def login_user(self, name): base_url = public_url(self) r = requests.post( base_url + 'hub/login', data={ 'username': name, 'password': name, }, allow_redirects=False, ) r.raise_for_status() assert r.cookies return r.cookies