def test__yield_for_all_futures():
loop = IOLoop()
loop.make_current()
@gen.coroutine
def several_steps():
value = 0
value += yield async_value(1)
value += yield async_value(2)
value += yield async_value(3)
raise gen.Return(value)
result = {}
def on_done(future):
result["value"] = future.result()
loop.stop()
loop.add_future(yield_for_all_futures(several_steps()), on_done)
try:
loop.start()
except KeyboardInterrupt as e:
print("keyboard interrupt")
assert 6 == result["value"]
loop.close()
def test_flexx_in_thread4():
""" Test threading starting server in other thread where it is created.
"""
res = []
loop = IOLoop()
loop.make_current()
app.create_server()
def try_start():
try:
app.stop()
app.start()
except RuntimeError:
res.append('start-fail')
else:
res.append('start-ok')
def main():
app.create_server()
try_start()
# Try to start server that was created in other thread -> fail
t = threading.Thread(target=try_start)
t.start()
t.join()
# Try to start in same thread as created -> ok
t = threading.Thread(target=main)
t.start()
t.join()
assert res == ['start-fail', 'start-ok']
def test_rebinding_ioloop():
""" Test recreating server objects, and its binding to the current ioloop.
"""
res = []
def add_res(i):
res.append(i)
# Create new ioloop
loop = IOLoop()
loop.make_current()
# Create new flexx server, which binds to that loop
server1 = app.create_server()
assert server1 is app.current_server()
#
assert loop is server1._loop
# Create new ioloop
loop = IOLoop()
loop.make_current()
# This is a new loop
assert loop is not server1._loop
# Create new flexx server, which binds to that loop
server2 = app.create_server()
assert server2 is app.current_server()
assert server1 is not server2
#
assert loop is server2._loop
def test_flexx_in_thread1():
""" Test threading and ioloop selection.
"""
def main():
loop2.make_current()
app.create_server()
# Create 3 loops, nr 2 is made current in the thread
loop1 = IOLoop()
loop2 = IOLoop()
loop3 = IOLoop()
loop1.make_current()
server1 = app.create_server()
t = threading.Thread(target=main)
t.start()
t.join()
server2 = app.current_server() # still current as set by the thread
loop3.make_current()
server3 = app.create_server()
assert server1._loop is loop1
assert server2._loop is loop2
assert server3._loop is loop3
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
worker_port, local_dir, services, name, memory_limit):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(scheduler_ip, scheduler_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services, name=name, memory_limit=memory_limit,
loop=loop) # pragma: no cover
@gen.coroutine # pragma: no cover
def start():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover
loop.add_callback(start) # pragma: no cover
try:
loop.start() # pragma: no cover
finally:
loop.stop()
loop.close(all_fds=True)
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port,
local_dir, services):
""" Function run by the Nanny when creating the worker """
from distributed import Worker
from tornado.ioloop import IOLoop
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(center_ip, center_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services)
@gen.coroutine
def start():
try:
yield worker._start()
except Exception as e:
logger.exception(e)
q.put(e)
else:
assert worker.port
q.put({'port': worker.port, 'dir': worker.local_dir})
loop.add_callback(start)
loop.start()
def loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
loop.stop()
loop.close()
def run_worker(q, ip, center_ip, center_port, ncores, nanny_port,
worker_port, local_dir, services, name):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(center_ip, center_port, ncores=ncores, ip=ip,
service_ports={'nanny': nanny_port}, local_dir=local_dir,
services=services, name=name) # pragma: no cover
@gen.coroutine # pragma: no cover
def start():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({'port': worker.port, 'dir': worker.local_dir}) # pragma: no cover
loop.add_callback(start) # pragma: no cover
with ignoring(KeyboardInterrupt):
loop.start() # pragma: no cover
def test_instance_pool(self):
instance1 = _Instance()
instance2 = _Instance()
pool = InstancePool([instance1, instance2])
pool.increment()
pool.increment()
self.assertEquals(instance1.value(), 1)
self.assertEquals(instance2.value(), 1)
pool.transaction(lambda i: i.increment())
pool.transaction(lambda i: i.increment())
self.assertEquals(instance1.value(), 2)
self.assertEquals(instance2.value(), 2)
@coroutine
def yield_tasks():
self.assertEquals((yield pool.await().increment()), 3)
self.assertEquals((yield pool.await().increment()), 3)
self.assertEquals(instance1.value(), 3)
self.assertEquals(instance2.value(), 3)
self.assertEquals((yield pool.await_transaction(lambda i: i.increment())), 4)
self.assertEquals((yield pool.await_transaction(lambda i: i.increment())), 4)
loop = IOLoop()
loop.make_current()
loop.run_sync(yield_tasks)
self.assertEquals(instance1.value(), 4)
self.assertEquals(instance2.value(), 4)
class XDebugServer(TCPServer):
"""Class to listen for xdebug requests"""
def __init__(self):
"""Constructor """
self.ioloop = IOLoop()
super(XDebugServer, self).__init__(io_loop=self.ioloop)
self.listen(9000)
# this is for cross thread communication
self.inport = Queue()
self.outport = Queue()
self._xdebug_connection = None
def listenfunc():
self.ioloop.make_current()
self.ioloop.start()
self.ioloop.close(all_fds=True)
self.listener_thread = threading.Thread(target=listenfunc)
self.listener_thread.daemon = True
self.listener_thread.start()
def handle_stream(self, stream, address):
"""Handle a connection
Only one connection at a time, for now
:stream: @todo
:address: @todo
:returns: @todo
"""
self._xdebug_connection = XDebugConnection(self, stream, address)
def run_command(self, command, data=None):
"""Send status
:returns: @todo
"""
self.inport.put("{} -i 1\0".format(str(command)))
return self.outport.get()
def stop(self):
"""Stop tornado event loop
:returns: @todo
"""
self.ioloop.stop()
self.listener_thread.join()
del self.ioloop
del self.listener_thread
def pristine_loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
try:
yield loop
finally:
loop.close(all_fds=True)
IOLoop.clear_instance()
def _run():
io_loop = IOLoop()
io_loop.make_current()
io_loop.add_callback(lambda : evt.set())
with mock.patch.dict(os.environ, env):
app = self._app = MockSingleUserServer()
app.initialize(args)
app.start()
def run_server(port, daemon='start'):
application = tornado.web.Application([
(r"/", TestHandler, {'port': port}),
])
ioloop = IOLoop()
ioloop.make_current()
SERVER_LOOPS.append(ioloop)
application.listen(port)
ioloop.start()
def atexit(self):
"""atexit callback"""
if self._atexit_ran:
return
self._atexit_ran = True
# run the cleanup step (in a new loop, because the interrupted one is unclean)
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self.cleanup)
def loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
sync(loop, loop.stop)
for i in range(5):
with ignoring(Exception):
loop.close(all_fds=True)
break
def _atexit(self):
if self._atexit_ran:
return
self._atexit_ran = True
self._stats_job.stop()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self._cleanup)
def run_center(port):
from distributed import Center
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1', port)
center.listen(port)
loop.start()
def test_func():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
cor = gen.coroutine(func)
try:
loop.run_sync(cor, timeout=timeout)
finally:
loop.stop()
loop.close(all_fds=True)
def run_nanny(port, center_port, **kwargs):
from distributed import Nanny
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Nanny('127.0.0.1', port, port + 1000, '127.0.0.1', center_port, **kwargs)
loop.run_sync(worker._start)
loop.start()
def _run():
asyncio.set_event_loop(asyncio.new_event_loop())
io_loop = IOLoop()
io_loop.make_current()
io_loop.add_callback(lambda: evt.set())
with mock.patch.dict(os.environ, env):
app = self._app = MockSingleUserServer()
app.initialize(args)
assert app.hub_auth.oauth_client_id
assert app.hub_auth.api_token
app.start()
def test_func():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
s, workers = loop.run_sync(lambda: start_cluster(ncores))
try:
loop.run_sync(lambda: cor(s, *workers), timeout=timeout)
finally:
loop.run_sync(lambda: end_cluster(s, workers))
loop.stop()
loop.close()
def main():
# Create fresh ioloop and make flexx use it
loop = IOLoop()
loop.make_current()
app.create_server()
# Create model and manipulate prop
model = MyModel1()
model.foo = 3
model.foo = 4
# Run mainloop for one iterartion
app.call_later(0, app.stop)
app.start()
def run_worker(port, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker("127.0.0.1", port, "127.0.0.1", center_port, **kwargs)
worker.start()
loop.start()
def run_worker(q, center_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Worker('127.0.0.1', center_port, ip='127.0.0.1', **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
loop.start()
def _initialize(queue):
result = None
try:
# create new IOLoop in the thread
io_loop = IOLoop()
# make it default for that thread
io_loop.make_current()
result = io_loop
io_loop.add_callback(queue.put, result)
io_loop.start()
except Exception as err: # pragma: no cover
result = err
finally: # pragma: no cover
queue.put(result)
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (KeyError, ValueError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def run_scheduler(q, scheduler_port=0, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
scheduler = Scheduler(loop=loop, validate=True, **kwargs)
done = scheduler.start(scheduler_port)
q.put(scheduler.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_worker(q, scheduler_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1',
loop=loop, validate=True, **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def current_loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
if loop._running:
sync(loop, loop.stop)
for i in range(5):
try:
loop.close(all_fds=True)
return
except Exception as e:
f = e
print(f)
IOLoop.clear_instance()
def test_awaitable(self):
instance = _Instance()
instance.increment()
self.assertEquals(instance.value(), 1)
awaitable = AwaitableInstance(instance)
@coroutine
def yield_tasks():
self.assertEquals((yield awaitable.increment()), 2)
self.assertEquals((yield awaitable.increment()), 3)
self.assertEquals((yield awaitable.increment()), 4)
self.assertEquals((yield awaitable.value()), 4)
loop = IOLoop()
loop.make_current()
loop.run_sync(yield_tasks)
self.assertEquals(instance.value(), 4)
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (KeyError, ValueError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def run_worker(q, scheduler_port, **kwargs):
from distributed import Worker
from tornado.ioloop import IOLoop, PeriodicCallback
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1',
loop=loop, validate=True, **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_scheduler(q, scheduler_port=0, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
scheduler = Scheduler(loop=loop, validate=True, **kwargs)
done = scheduler.start(scheduler_port)
q.put(scheduler.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def current_loop():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
yield loop
if loop._running:
sync(loop, loop.stop)
for i in range(5):
try:
loop.close(all_fds=True)
return
except Exception as e:
f = e
print(f)
IOLoop.clear_instance()
class LoopAndGroup:
def __init__(self, quit_after=None) -> None:
self.io_loop = IOLoop()
self.io_loop.make_current()
self.group = _CallbackGroup(self.io_loop)
if quit_after is not None:
self.io_loop.call_later(quit_after / 1000.0,
lambda: self.io_loop.stop())
def __exit__(self, type, value, traceback):
run(self.io_loop)
self.io_loop.close()
def __enter__(self):
return self
def run_nanny(q, center_port, **kwargs):
from distributed import Nanny
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
with log_errors():
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
worker = Nanny('127.0.0.1', center_port, ip='127.0.0.1', **kwargs)
loop.run_sync(lambda: worker._start(0))
q.put(worker.port)
try:
loop.start()
finally:
loop.close(all_fds=True)
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
worker_port, local_dir, services, name, memory_limit,
reconnect, resources, validate):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(scheduler_ip,
scheduler_port,
ncores=ncores,
ip=ip,
service_ports={'nanny': nanny_port},
local_dir=local_dir,
services=services,
name=name,
memory_limit=memory_limit,
reconnect=reconnect,
validate=validate,
resources=resources,
loop=loop) # pragma: no cover
@gen.coroutine # pragma: no cover
def run():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({
'port': worker.port,
'dir': worker.local_dir
}) # pragma: no cover
while worker.status != 'closed':
yield gen.sleep(0.1)
logger.info("Worker closed")
try:
loop.run_sync(run)
finally:
loop.stop()
loop.close(all_fds=True)
def test_base_server() -> None:
app = BokehTornado(Application())
httpserver = HTTPServer(app)
httpserver.start()
loop = IOLoop()
loop.make_current()
server = BaseServer(loop, app, httpserver)
server.start()
assert server.io_loop == loop
assert server._tornado.io_loop == loop
httpserver.stop()
server.stop()
server.io_loop.close()
def prepare_browser(project, stage, io_loop, show_url):
from tornado.ioloop import IOLoop
from conda_kapsel.internal.ui_server import UIServer, UIServerDoneEvent
result_holder = {}
old_current_loop = None
try:
old_current_loop = IOLoop.current()
if io_loop is None:
io_loop = IOLoop()
io_loop.make_current()
if show_url is None:
show_url = _default_show_url
def event_handler(event):
if isinstance(event, UIServerDoneEvent):
result_holder['result'] = event.result
io_loop.stop()
server = UIServer(project,
stage,
event_handler=event_handler,
io_loop=io_loop)
try:
print("# Configure the project at {url} to continue...".format(
url=server.url))
show_url(server.url)
io_loop.start()
finally:
server.unlisten()
finally:
if old_current_loop is not None:
old_current_loop.make_current()
if 'result' in result_holder:
return result_holder['result']
else:
from conda_kapsel.prepare import PrepareFailure
# this pretty much only happens in unit tests.
return PrepareFailure(logs=[],
statuses=(),
errors=["Browser UI main loop was stopped."],
environ=stage.environ,
overrides=stage.overrides)
def do_test(dirname):
io_loop = IOLoop()
io_loop.make_current()
events = []
def event_handler(event):
events.append(event)
project = Project(dirname)
local_state_file = LocalStateFile.load_for_directory(dirname)
requirement = EnvVarRequirement(registry=project.plugin_registry, env_var="FOO")
status = requirement.check_status(dict(), local_state_file, 'default', UserConfigOverrides())
context = ConfigurePrepareContext(dict(), local_state_file, 'default', UserConfigOverrides(), [status])
server = UIServer(project, _no_op_prepare(context), event_handler, io_loop)
# do a get so that _requirements_by_id below exists
get_response = http_get(io_loop, server.url)
assert 200 == get_response.code
req_id = list(server._application._requirements_by_id.keys())[0]
if '%s' in name_template:
name = name_template % req_id
else:
name = name_template
encoder = MultipartEncoder({name: 'bloop'})
body = encoder.to_string()
headers = {'Content-Type': encoder.content_type}
post_response = http_post(io_loop, server.url, body=body, headers=headers)
# we just ignore bad form names, because they are assumed
# to be some sort of hostile thing. we shouldn't ever
# generate them on purpose.
assert 200 == post_response.code
server.unlisten()
assert len(events) == 1
assert isinstance(events[0], UIServerDoneEvent)
out, err = capsys.readouterr()
assert out == ""
assert err == expected_err
class LoopOverhead(object):
"""
These are not distributed benchmarks per se, but help assessing
Tornado's loop management overhead for other benchmarks.
"""
def setup(self):
self.loop = IOLoop()
self.loop.make_current()
def time_loop_start_stop(self):
self.loop.add_callback(self.loop.stop)
self.loop.start()
@gen.coroutine
def _empty_coro(self):
pass
def time_loop_run_sync(self):
run_sync(self.loop, self._empty_coro)
def run_scheduler(q, center_port=None, **kwargs):
from distributed import Scheduler
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = ('127.0.0.1', center_port) if center_port else None
scheduler = Scheduler(center=center, **kwargs)
scheduler.listen(0)
if center_port:
loop.run_sync(scheduler.sync_center)
done = scheduler.start(0)
q.put(scheduler.port)
loop.start()
def test_restarting():
""" Test stopping and starting the ioloop.
"""
res = []
def add_res(i):
res.append(i)
def try_start():
try:
app.start()
except RuntimeError:
res.append('RTE')
# Create new ioloop always
loop = IOLoop()
loop.make_current()
# Make Flexx use it
server = app.create_server()
app.call_later(0, add_res, 1)
app.call_later(0, add_res, 2)
app.call_later(
0, app.stop) # actually, just calling stop() would work as well
app.start()
assert server._running == False
app.call_later(0, try_start) # test that cannot start twice
app.call_later(0, add_res, 3)
app.call_later(0, add_res, 4)
app.call_later(0, app.stop)
app.start()
assert server._running == False
app.call_later(0, try_start) # test that cannot start twice
app.call_later(0, add_res, 5)
app.call_later(0, add_res, 6)
app.call_later(0, app.stop)
app.start()
assert server._running == False
assert res == [1, 2, 'RTE', 3, 4, 'RTE', 5, 6]
def do_test(dirname):
io_loop = IOLoop()
io_loop.make_current()
events = []
def event_handler(event):
events.append(event)
local_state_file = LocalStateFile.load_for_directory(dirname)
value = local_state_file.get_value(['variables', 'FOO'])
assert value is None
project = Project(dirname)
requirement = EnvVarRequirement(registry=project.plugin_registry, env_var="FOO")
status = requirement.check_status(dict(), local_state_file, 'default', UserConfigOverrides())
context = ConfigurePrepareContext(dict(), local_state_file, 'default', UserConfigOverrides(), [status])
server = UIServer(project, _no_op_prepare(context), event_handler, io_loop)
get_response = http_get(io_loop, server.url)
print(repr(get_response))
soup = BeautifulSoup(get_response.body, _BEAUTIFUL_SOUP_BACKEND)
field = soup.find_all("input", attrs={'type': 'text'})[0]
assert 'name' in field.attrs
encoder = MultipartEncoder({field['name']: 'bloop'})
body = encoder.to_string()
headers = {'Content-Type': encoder.content_type}
post_response = http_post(io_loop, server.url, body=body, headers=headers)
print(repr(post_response))
server.unlisten()
assert len(events) == 1
assert isinstance(events[0], UIServerDoneEvent)
value = local_state_file.get_value(['variables', 'FOO'])
assert 'bloop' == value
def pristine_loop():
"""
Builds a clean IOLoop for using as a background request.
Courtesy of Dask Distributed
"""
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (ValueError, KeyError, RuntimeError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def run_center(q):
from distributed import Center
from tornado.ioloop import IOLoop, PeriodicCallback
import logging
IOLoop.clear_instance()
loop = IOLoop(); loop.make_current()
PeriodicCallback(lambda: None, 500).start()
logging.getLogger("tornado").setLevel(logging.CRITICAL)
center = Center('127.0.0.1')
while True:
try:
center.listen(0)
break
except Exception as e:
logging.info("Could not start center on port. Retrying",
exc_info=True)
q.put(center.port)
loop.start()
class ControlThread(Thread):
def __init__(self, **kwargs):
Thread.__init__(self, **kwargs)
self.io_loop = IOLoop(make_current=False)
self.pydev_do_not_trace = True
self.is_pydev_daemon_thread = True
def run(self):
self.io_loop.make_current()
try:
self.io_loop.start()
finally:
self.io_loop.close()
def stop(self):
"""Stop the thread.
This method is threadsafe.
"""
self.io_loop.add_callback(self.io_loop.stop)
def run_worker_fork(q, scheduler_addr, ncores, nanny_port, worker_ip,
worker_port, local_dir, **kwargs):
"""
Create a worker by forking.
"""
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(scheduler_addr,
ncores=ncores,
service_ports={'nanny': nanny_port},
local_dir=local_dir,
**kwargs) # pragma: no cover
@gen.coroutine # pragma: no cover
def run():
try: # pragma: no cover
yield worker._start((worker_ip, worker_port)) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({
'address': worker.address,
'dir': worker.local_dir
}) # pragma: no cover
yield worker.wait_until_closed()
logger.info("Worker closed")
try:
loop.run_sync(run)
except TimeoutError:
logger.info("Worker timed out")
finally:
loop.stop()
loop.close(all_fds=True)
def test_more_stopping():
""" Test calling stop multiple times.
"""
# This is why you want to create new IOLoop instances for each test
# Create new ioloop and make Flexx use it
loop = IOLoop()
loop.make_current()
server = app.create_server()
app.stop() # triggers event to stop
app.start()
app.stop() # Extra stop - pending stop event
# Which means the next stop does hardly block
t0 = time.time()
app.call_later(0.2, app.stop)
app.start()
assert time.time() - t0 < 0.1
loop = IOLoop()
loop.make_current()
server = app.create_server()
# But stops dont stack
app.stop()
app.stop()
app.stop()
app.stop()
# Flush all stops ...
app.stop()
app.start()
# ... so that we now have an expected loop
t0 = time.time()
app.call_later(0.2, app.stop)
app.start()
assert time.time() - t0 >= 0.1
def test_func():
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
s, workers = loop.run_sync(
lambda: start_cluster(ncores, loop, Worker=Worker))
args = [s] + workers
if client:
e = Client((s.ip, s.port), loop=loop, start=False)
loop.run_sync(e._start)
args = [e] + args
try:
loop.run_sync(lambda: cor(*args), timeout=timeout)
finally:
if client:
loop.run_sync(e._shutdown)
loop.run_sync(lambda: end_cluster(s, workers))
loop.stop()
loop.close(all_fds=True)
def run_worker_fork(q, ip, scheduler_ip, scheduler_port, ncores, nanny_port,
worker_port, local_dir, services, name, memory_limit):
""" Function run by the Nanny when creating the worker """
from distributed import Worker # pragma: no cover
from tornado.ioloop import IOLoop # pragma: no cover
IOLoop.clear_instance() # pragma: no cover
loop = IOLoop() # pragma: no cover
loop.make_current() # pragma: no cover
worker = Worker(scheduler_ip,
scheduler_port,
ncores=ncores,
ip=ip,
service_ports={'nanny': nanny_port},
local_dir=local_dir,
services=services,
name=name,
memory_limit=memory_limit,
loop=loop) # pragma: no cover
@gen.coroutine # pragma: no cover
def start():
try: # pragma: no cover
yield worker._start(worker_port) # pragma: no cover
except Exception as e: # pragma: no cover
logger.exception(e) # pragma: no cover
q.put(e) # pragma: no cover
else:
assert worker.port # pragma: no cover
q.put({
'port': worker.port,
'dir': worker.local_dir
}) # pragma: no cover
loop.add_callback(start) # pragma: no cover
try:
loop.start() # pragma: no cover
finally:
loop.stop()
loop.close(all_fds=True)
def do_test(dirname):
io_loop = IOLoop()
io_loop.make_current()
events = []
def event_handler(event):
events.append(event)
project = Project(dirname)
local_state_file = LocalStateFile.load_for_directory(dirname)
context = ConfigurePrepareContext(dict(), local_state_file, 'default', UserConfigOverrides(), [])
server = UIServer(project, _no_op_prepare(context), event_handler, io_loop)
get_response = http_get(io_loop, server.url)
print(repr(get_response))
post_response = http_post(io_loop, server.url, body="")
print(repr(post_response))
server.unlisten()
assert len(events) == 1
assert isinstance(events[0], UIServerDoneEvent)
def test__ioloop_not_forcibly_stopped():
# Issue #5494
application = Application()
loop = IOLoop()
loop.make_current()
server = Server(application, ioloop=loop)
server.start()
result = []
def f():
server.unlisten()
server.stop()
# If server.stop() were to stop the Tornado IO loop,
# g() wouldn't be called and `result` would remain empty.
loop.add_timeout(timedelta(seconds=0.01), g)
def g():
result.append(None)
loop.stop()
loop.add_callback(f)
loop.start()
assert result == [None]
class Connect(object):
"""
Test overhead of connect() and Comm.close().
"""
N_CONNECTS = 100
def setup(self):
self.loop = IOLoop()
self.loop.make_current()
@gen.coroutine
def _handle_comm(self, comm):
yield comm.close()
@gen.coroutine
def _connect_close(self, addr):
comm = yield connect(addr)
yield comm.close()
@gen.coroutine
def _main(self, address):
listener = listen(address, self._handle_comm)
listener.start()
yield [
self._connect_close(listener.contact_address)
for i in range(self.N_CONNECTS)
]
listener.stop()
def _time_connect(self, address):
run_sync(self.loop, partial(self._main, address))
def time_tcp_connect(self):
self._time_connect('tcp://127.0.0.1')
def time_inproc_connect(self):
self._time_connect('inproc://')
def run(self, loop=None):
'''
Start servicing the Tornado event loop.
'''
if not loop:
loop = IOLoop()
loop.make_current()
# bind the socket
self.listen(self._port, self._address)
logger.info('Pensive started on {}:{}'.format(self._address or '*',
self._port))
try:
loop.start()
except KeyboardInterrupt:
pass
loop.stop()
loop.close()
logger.info('Pensive stopped')
def main():
""" Main function for this module. Load and run tests.
Each test run a game and checks game messages and phases against an expected game data file.
Current tested gama data files are JSON files located into folder diplomacy/tests/network.
"""
parser = argparse.ArgumentParser(description='Run test cases against an external server to connect.')
parser.add_argument('--port', type=int, required=True,
help='run on the given port (required)')
parser.add_argument('--cases', action='append',
help="Run given cases. "
"Each case <C> must match a test case file <C>.json located in diplomacy.tests.network. "
"If not provided, all available cases are run.")
args = parser.parse_args()
io_loop = IOLoop()
io_loop.make_current()
@gen.coroutine
def run():
""" Run all tests consecutively in one call. """
tests = set(args.cases) if args.cases else {'1', '2', '3'}
for test_case in list(sorted(tests)):
yield launch_case('%s.json' % test_case, args.port, io_loop)
io_loop.run_sync(run)
def __init__(self, application, **server_kwargs):
loop = IOLoop()
loop.make_current()
server_kwargs['io_loop'] = loop
self._server = Server(application, **server_kwargs)
def _run(
cls,
worker_kwargs,
worker_start_args,
silence_logs,
init_result_q,
child_stop_q,
uid,
env,
config,
Worker,
): # pragma: no cover
os.environ.update(env)
dask.config.set(config)
try:
from dask.multiprocessing import initialize_worker_process
except ImportError: # old Dask version
pass
else:
initialize_worker_process()
if silence_logs:
logger.setLevel(silence_logs)
IOLoop.clear_instance()
loop = IOLoop()
loop.make_current()
worker = Worker(**worker_kwargs)
async def do_stop(timeout=5, executor_wait=True):
try:
await worker.close(
report=False,
nanny=False,
executor_wait=executor_wait,
timeout=timeout,
)
finally:
loop.stop()
def watch_stop_q():
"""
Wait for an incoming stop message and then stop the
worker cleanly.
"""
while True:
try:
msg = child_stop_q.get(timeout=1000)
except Empty:
pass
else:
child_stop_q.close()
assert msg.pop("op") == "stop"
loop.add_callback(do_stop, **msg)
break
t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
t.daemon = True
t.start()
async def run():
"""
Try to start worker and inform parent of outcome.
"""
try:
await worker
except Exception as e:
logger.exception("Failed to start worker")
init_result_q.put({"uid": uid, "exception": e})
init_result_q.close()
else:
try:
assert worker.address
except ValueError:
pass
else:
init_result_q.put(
{
"address": worker.address,
"dir": worker.local_directory,
"uid": uid,
}
)
init_result_q.close()
await worker.finished()
logger.info("Worker closed")
try:
loop.run_sync(run)
except TimeoutError:
# Loop was stopped before wait_until_closed() returned, ignore
pass
except KeyboardInterrupt:
pass
class CompatibilityTests(unittest.TestCase):
def setUp(self):
self.saved_signals = save_signal_handlers()
self.io_loop = IOLoop()
self.io_loop.make_current()
self.reactor = AsyncioSelectorReactor()
def tearDown(self):
self.reactor.disconnectAll()
self.io_loop.clear_current()
self.io_loop.close(all_fds=True)
restore_signal_handlers(self.saved_signals)
def start_twisted_server(self):
class HelloResource(Resource):
isLeaf = True
def render_GET(self, request):
return b"Hello from twisted!"
site = Site(HelloResource())
port = self.reactor.listenTCP(0, site, interface='127.0.0.1')
self.twisted_port = port.getHost().port
def start_tornado_server(self):
class HelloHandler(RequestHandler):
def get(self):
self.write("Hello from tornado!")
app = Application([('/', HelloHandler)], log_function=lambda x: None)
server = HTTPServer(app)
sock, self.tornado_port = bind_unused_port()
server.add_sockets([sock])
def run_reactor(self):
# In theory, we can run the event loop through Tornado,
# Twisted, or asyncio interfaces. However, since we're trying
# to avoid installing anything as the global event loop, only
# the twisted interface gets everything wired up correectly
# without extra hacks. This method is a part of a
# no-longer-used generalization that allowed us to test
# different combinations.
self.stop_loop = self.reactor.stop
self.stop = self.reactor.stop
self.reactor.run()
def tornado_fetch(self, url, runner):
client = AsyncHTTPClient()
fut = client.fetch(url)
fut.add_done_callback(lambda f: self.stop_loop())
runner()
return fut.result()
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request(b'GET', utf8(url))
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(failure):
if hasattr(self, 'stop_loop'):
self.stop_loop()
elif failure is not None:
# loop hasn't been initialized yet; try our best to
# get an error message out. (the runner() interaction
# should probably be refactored).
try:
failure.raiseException()
except:
logging.error('exception before starting loop',
exc_info=True)
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return b''.join(chunks)
def twisted_coroutine_fetch(self, url, runner):
body = [None]
@gen.coroutine
def f():
# This is simpler than the non-coroutine version, but it cheats
# by reading the body in one blob instead of streaming it with
# a Protocol.
client = Agent(self.reactor)
response = yield client.request(b'GET', utf8(url))
with warnings.catch_warnings():
# readBody has a buggy DeprecationWarning in Twisted 15.0:
# https://twistedmatrix.com/trac/changeset/43379
warnings.simplefilter('ignore', category=DeprecationWarning)
body[0] = yield readBody(response)
self.stop_loop()
self.io_loop.add_callback(f)
runner()
return body[0]
def testTwistedServerTornadoClientReactor(self):
self.start_twisted_server()
response = self.tornado_fetch(
'http://127.0.0.1:%d' % self.twisted_port, self.run_reactor)
self.assertEqual(response.body, b'Hello from twisted!')
def testTornadoServerTwistedClientReactor(self):
self.start_tornado_server()
response = self.twisted_fetch(
'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, b'Hello from tornado!')
def testTornadoServerTwistedCoroutineClientReactor(self):
self.start_tornado_server()
response = self.twisted_coroutine_fetch(
'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, b'Hello from tornado!')
class MockHub(JupyterHub):
"""Hub with various mock bits"""
db_file = None
confirm_no_ssl = True
last_activity_interval = 2
@default('subdomain_host')
def _subdomain_host_default(self):
return os.environ.get('JUPYTERHUB_TEST_SUBDOMAIN_HOST', '')
@default('ip')
def _ip_default(self):
return '127.0.0.1'
@default('authenticator_class')
def _authenticator_class_default(self):
return MockPAMAuthenticator
@default('spawner_class')
def _spawner_class_default(self):
return MockSpawner
def init_signal(self):
pass
def start(self, argv=None):
self.db_file = NamedTemporaryFile()
self.pid_file = NamedTemporaryFile(delete=False).name
self.db_url = self.db_file.name
evt = threading.Event()
@gen.coroutine
def _start_co():
assert self.io_loop._running
# put initialize in start for SQLAlchemy threading reasons
yield super(MockHub, self).initialize(argv=argv)
# add an initial user
user = orm.User(name='user')
self.db.add(user)
self.db.commit()
yield super(MockHub, self).start()
yield self.hub.server.wait_up(http=True)
self.io_loop.add_callback(evt.set)
def _start():
self.io_loop = IOLoop()
self.io_loop.make_current()
self.io_loop.add_callback(_start_co)
self.io_loop.start()
self._thread = threading.Thread(target=_start)
self._thread.start()
ready = evt.wait(timeout=10)
assert ready
def stop(self):
super().stop()
self._thread.join()
IOLoop().run_sync(self.cleanup)
# ignore the call that will fire in atexit
self.cleanup = lambda: None
self.db_file.close()
def login_user(self, name):
base_url = public_url(self)
r = requests.post(
base_url + 'hub/login',
data={
'username': name,
'password': name,
},
allow_redirects=False,
)
r.raise_for_status()
assert r.cookies
return r.cookies
