def run(self):
#logging.basicConfig(filename='runserver.log',level=logging.DEBUG)
IOLoop.clear_current()
IOLoop.clear_instance()
self.io_loop = IOLoop.instance()
ws_connection_handler = WebSocketConnectionHandler()
results_cache = ResultCache()
tornado_app = Application(handlers=[(r"/ws/(.*)", AsyncRunHandler, {
'connection_handler': ws_connection_handler,
'result_cache': results_cache,
'io_loop': self.io_loop,
}), (r"/ping", PingRequestHandler)])
self.http_server = HTTPServer(tornado_app)
try:
self.http_server.listen(port=SERVER_PORT, address=SERVER_ADDR)
if not self.io_loop._running:
print('Running Server Loop')
self.io_loop.start()
else:
print("IOLoop already running")
except OSError:
print("Server is already running!")
except KeyboardInterrupt: # SIGINT, SIGTERM
print('Closing Server Loop')
self.http_server.close_all_connections()
self.io_loop.stop()
def tearDown(self):
self.db.close()
try:
os.remove(self.temp_db)
except:
pass
IOLoop.clear_current()
def init_process(self):
# IOLoop cannot survive a fork or be shared across processes
# in any way. When multiple processes are being used, each process
# should create its own IOLoop. We should clear current IOLoop
# if exists before os.fork.
IOLoop.clear_current()
super().init_process()
def init_process(self):
# IOLoop cannot survive a fork or be shared across processes
# in any way. When multiple processes are being used, each process
# should create its own IOLoop. We should clear current IOLoop
# if exists before os.fork.
IOLoop.clear_current()
super().init_process()
def tearDown(self):
self.db.close()
try:
os.remove(self.temp_db)
except:
pass
IOLoop.clear_current()
def io_loop() -> AsyncIOLoop:
loop = IOLoop()
loop.make_current()
try:
yield loop
finally:
loop.clear_current()
loop.close(all_fds=True)
def start(self):
self.client = TCPClient()
self.pcb = PeriodicCallback(self.send, 1000.0 / self.n)
self.pcb.start()
IOLoop.current().call_later(self.duration + 0.5, self.stop)
IOLoop.current().start()
IOLoop.clear_current()
def test_sync_closed_loop():
loop = IOLoop.current()
loop.close()
IOLoop.clear_current()
IOLoop.clear_instance()
with pytest.raises(RuntimeError) as exc_info:
sync(loop, inc, 1)
exc_info.match("IOLoop is clos(ed|ing)")
def start(self): try: app = ftApplication(self.HANDLERS) server = HTTPServer(app) server.bind(self.PORT) server.start(self.PROCESS_NUM) IOLoop.clear_current().start() except Exception, msg: print "[tornado server start Exceptions : %s]" % msg
def start(self):
self.client = TCPClient()
self.pcb = PeriodicCallback(self.send, 1000.0 / self.n)
self.pcb.start()
IOLoop.current().call_later(self.duration + 0.5, self.stop)
IOLoop.current().start()
IOLoop.clear_current()
def test_sync_closed_loop():
loop = IOLoop.current()
loop.close()
IOLoop.clear_current()
IOLoop.clear_instance()
with pytest.raises(RuntimeError) as exc_info:
sync(loop, inc, 1)
exc_info.match("IOLoop is clos(ed|ing)")
def _atexit(self):
if self._atexit_ran:
return
self._atexit_ran = True
self._stats_job.stop()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self._cleanup)
def start(self):
old_current = IOLoop.current(instance=False)
try:
self._setup_logging()
self.reactor.run()
finally:
if old_current is None:
IOLoop.clear_current()
else:
old_current.make_current()
def atexit(self):
"""atexit callback"""
if self._atexit_ran:
return
self._atexit_ran = True
# run the cleanup step (in a new loop, because the interrupted one is unclean)
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self.cleanup)
def _atexit(self):
if self._atexit_ran:
return
self._atexit_ran = True
self._stats_job.stop()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self._cleanup)
def awaitTermination(self, timeout=None):
"""Wait for context to stop.
:param float timeout: in seconds
"""
if timeout is not None:
IOLoop.current().call_later(timeout, self.stop)
IOLoop.current().start()
IOLoop.clear_current()
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
try:
yield loop
finally:
loop.close(all_fds=True)
IOLoop.clear_instance()
IOLoop.clear_current()
def start(self):
old_current = IOLoop.current(instance=False)
try:
self._setup_logging()
self.make_current()
self.reactor.run()
finally:
if old_current is None:
IOLoop.clear_current()
else:
old_current.make_current()
def start(self):
old_current = IOLoop.current(instance=False)
try:
self._setup_logging()
self.make_current()
self.asyncio_loop.run_forever()
finally:
if old_current is None:
IOLoop.clear_current()
else:
old_current.make_current()
def start(self):
"""Start IO loop and try to connect to the server"""
self.ioloop.make_current()
self.connect()
self.periodic.start()
self.ioloop.call_later(1, self.periodic_callback)
logger.debug('Starting IOLoop')
self.ioloop.start()
logger.debug('IOLoop ended')
IOLoop.clear_current()
def start(self):
old_current = IOLoop.current(instance=False)
try:
self._setup_logging()
self.make_current()
self.asyncio_loop.run_forever()
finally:
if old_current is None:
IOLoop.clear_current()
else:
old_current.make_current()
def wrapper(*args, **kwargs):
loop = None
try:
IOLoop.clear_current()
loop = asyncio.new_event_loop()
loop.run_until_complete(fn(*args, **kwargs))
finally:
if loop is not None:
loop.close()
IOLoop.clear_current()
asyncio.set_event_loop(None)
class _Executor(Thread):
"""
Tasks executor. Task is executed in ioloop for easier stopping it. Subprocess based tasks are killed external
"""
def __init__(self, task, number, *args, **kwargs):
super(_Executor, self).__init__(*args, **kwargs)
self.ioloop = None
self.task = task
self.number = number
def run(self):
if self.task.cancelled:
return
self.task.set_executor(executor=self)
self.ioloop = IOLoop()
self.ioloop.make_current()
self.ioloop.add_callback(self.execute)
self.ioloop.start()
self.task.clear()
self.ioloop.clear_current()
async def execute(self):
"""
Update task and stop ioloop
"""
try:
await self.task()
await self.task.post_run()
except subprocess.CalledProcessError as exception:
log.warning('%s', exception)
except:
log.exception("Exception while executing task on worker %s",
self.number)
finally:
self.ioloop.stop()
self.task.finish_time = int(time.time())
def stop(self):
"""
Stop task. Important especially for Subprocess based tasks
"""
self.task.kill()
# As Subprocess based tasks generate traffic only using external tool, they should exit gracefully
if not isinstance(self.task,
(CommandTask, NmapPortInfoTask, PortScanTask)):
self.ioloop.stop()
def __str__(self):
return str(self.task)
def wrapper(*args, **kwargs):
loop = None
try:
IOLoop.clear_current()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(fn(*args, **kwargs))
finally:
if loop is not None:
loop.close()
IOLoop.clear_current()
asyncio.set_event_loop(None)
def setup_event_loop(self):
self.evtloop = asyncio.new_event_loop()
asyncio.set_event_loop(self.evtloop)
IOLoop.clear_current()
if hasattr(IOLoop, '_current'):
del IOLoop._current
if self.loop is None:
if not IOLoop.current(instance=False):
self.loop = IOLoop.current() #IOLoop()
else:
self.loop = IOLoop.current()
def main():
"""Main function"""
logger.debug('Starting')
logger.debug(u'pid={}'.format(os.getpid()))
try:
__once__ = OnlyOnce()
logger.debug(str(__once__))
except OnlyOnceException:
logger.debug('Tunnel already running, exiting')
sys.exit(0)
io_loop = IOLoop(make_current=False)
library_cache = LibraryCache()
kodi = KodiInterface(library_cache)
handler = Handler(kodi)
monitor = CustomMonitor(kodi)
player = CustomPlayer()
player.set_io_loop(io_loop)
player.set_kodi(kodi)
tunnel = Tunnel(io_loop, KODI_CONNECT_URL, kodi, handler)
tunnel_thread = TunnelThread(tunnel)
tunnel_thread.start()
async_tunnel = tunnel.get_async_tunnel()
player.set_async_tunnel(async_tunnel)
try:
while not monitor.abortRequested():
# Sleep/wait for abort for 3 seconds
if monitor.waitForAbort(3):
# Abort was requested while waiting. We should exit
break
except KeyboardInterrupt:
logger.debug('Interrupted')
logger.debug('Stopping __once__')
__once__.stop()
logger.debug('Stopping Tunnel')
tunnel_thread.stop()
logger.debug('Joining Tunnel Thread')
tunnel_thread.join()
logger.debug('Clearing main IOLoop')
IOLoop.clear_current()
logger.debug('Stopping concurrent ThreadPool')
concurrent.futures.thread._python_exit() # pylint: disable=protected-access
logger.debug('Exit')
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except ValueError:
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def single_run_ws(url, delay=0, size=0, msgs=1):
"""Time a single websocket run"""
buf = hexlify(os.urandom(size // 2)).decode('ascii')
msg = json.dumps({'delay': delay, 'data': buf})
async def go():
ws = await websocket_connect(url.replace('http', 'ws') + '/ws')
for i in range(msgs):
ws.write_message(msg)
await ws.read_message()
asyncio.set_event_loop(asyncio.new_event_loop())
IOLoop.clear_current()
loop = IOLoop(make_current=True)
loop.run_sync(go)
def single_run_ws(url, delay=0, size=0, msgs=1):
"""Time a single websocket run"""
buf = hexlify(os.urandom(size // 2)).decode('ascii')
msg = json.dumps({'delay': delay, 'data': buf})
async def go():
ws = await websocket_connect(url.replace('http', 'ws') + '/ws')
for i in range(msgs):
ws.write_message(msg)
await ws.read_message()
asyncio.set_event_loop(asyncio.new_event_loop())
IOLoop.clear_current()
loop = IOLoop(make_current=True)
loop.run_sync(go)
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (KeyError, ValueError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def pristine_loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (KeyError, ValueError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def start(self):
old_current = IOLoop.current(instance=False)
try:
old_asyncio = asyncio.get_event_loop()
except RuntimeError:
old_asyncio = None
try:
self._setup_logging()
self.make_current()
# This is automatic in 3.5, but must be done manually in 3.4.
asyncio.set_event_loop(self.asyncio_loop)
self.asyncio_loop.run_forever()
finally:
if old_current is None:
IOLoop.clear_current()
else:
old_current.make_current()
asyncio.set_event_loop(old_asyncio)
def start(self):
old_current = IOLoop.current(instance=False)
try:
old_asyncio = asyncio.get_event_loop()
except RuntimeError:
old_asyncio = None
try:
self._setup_logging()
self.make_current()
# This is automatic in 3.5, but must be done manually in 3.4.
asyncio.set_event_loop(self.asyncio_loop)
self.asyncio_loop.run_forever()
finally:
if old_current is None:
IOLoop.clear_current()
else:
old_current.make_current()
asyncio.set_event_loop(old_asyncio)
def run():
io_loop = IOLoop(make_current=True)
app = Application([("/", RootHandler)])
port = random.randrange(options.min_port, options.max_port)
app.listen(port, address='127.0.0.1')
signal.signal(signal.SIGCHLD, handle_sigchld)
args = ["ab"]
args.extend(["-n", str(options.n)])
args.extend(["-c", str(options.c)])
if options.keepalive:
args.append("-k")
if options.quiet:
# just stops the progress messages printed to stderr
args.append("-q")
args.append("http://127.0.0.1:%d/" % port)
subprocess.Popen(args)
io_loop.start()
io_loop.close()
io_loop.clear_current()
def run():
io_loop = IOLoop(make_current=True)
app = Application([("/", RootHandler)])
port = random.randrange(options.min_port, options.max_port)
app.listen(port, address='127.0.0.1')
signal.signal(signal.SIGCHLD, handle_sigchld)
args = ["ab"]
args.extend(["-n", str(options.n)])
args.extend(["-c", str(options.c)])
if options.keepalive:
args.append("-k")
if options.quiet:
# just stops the progress messages printed to stderr
args.append("-q")
args.append("http://127.0.0.1:%d/" % port)
subprocess.Popen(args)
io_loop.start()
io_loop.close()
io_loop.clear_current()
def loop():
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
yield loop
if loop._running:
sync(loop, loop.stop)
for i in range(5):
try:
loop.close(all_fds=True)
IOLoop.clear_instance()
break
except Exception as e:
f = e
else:
print(f)
IOLoop.clear_instance()
IOLoop.clear_current()
def pristine_loop():
"""
Builds a clean IOLoop for using as a background request.
Courtesy of Dask Distributed
"""
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
assert IOLoop.current() is loop
try:
yield loop
finally:
try:
loop.close(all_fds=True)
except (ValueError, KeyError, RuntimeError):
pass
IOLoop.clear_instance()
IOLoop.clear_current()
def run_server(port, password, dbfile):
"""Start up the HTTP server. If Tornado is available it will be used, else
fall back to the Flask debug server.
"""
app.config['PASSWORD'] = password
app.config['DBFILE'] = dbfile
if USE_TORNADO:
# When running inside an IPython Notebook, the IOLoop
# can inherit a stale instance from the parent process,
# so clear that.
# https://github.com/adamgreig/sheepdog/issues/15
# Thanks @minrk!
if hasattr(IOLoop, '_instance'):
del IOLoop._instance
IOLoop.clear_current()
HTTPServer(WSGIContainer(app)).listen(port)
IOLoop.instance().start()
else:
app.run(host='0.0.0.0', port=port)
def create_new_io_loop():
"""Returns new IOLoop and doesn't set it current.
It's definetely usefull for Sync services to not to stop
IOLoop.current or IOLoop.instance"""
# get a current IOLoop to store
# we don't want to get IOLoop.instance
old = IOLoop.current(instance=False)
# create new IOLoop and sets it current
io_loop = IOLoop()
# io_loop is set curent now, we need to avoid it
# by replacing the current value with `old`
if old:
# `old` was a current IOLoop, make it again
old.make_current()
else:
# this thread had no current IOLoop
# so make it cleat again
IOLoop.clear_current()
return io_loop
def run_in_pool(_pid, _f, _has_context, context, *args, **kwargs):
# globals from the parent process in the
# IOLoop so clear them.
subprocess = os.getpid() != _pid
if subprocess and IOLoop.current(False):
LOGGER.debug("clearing tornado globals")
IOLoop.clear_current()
IOLoop.clear_instance()
LOGGER.debug("running %s %s", os.getpid(), context)
if _has_context:
kwargs[_has_context] = context
result = _f(*args, **kwargs)
if not subprocess:
return result
if isinstance(result, Future):
LOGGER.debug('running up tornado to complete')
def done(*args, **kwargs):
LOGGER.debug('stopping tornado')
IOLoop.current().stop()
result.add_done_callback(done)
IOLoop.current().start()
result = result.result()
return context, result
def tearDown(self):
super(TestFutureSocket, self).tearDown()
if self.loop:
self.loop.close(all_fds=True)
IOLoop.clear_current()
IOLoop.clear_instance()
def setUp(self):
self._saved_signals = save_signal_handlers()
self._io_loop = IOLoop(make_current=True)
self._reactor = TornadoReactor()
IOLoop.clear_current()
def setUp(self):
self.io_loop = None
IOLoop.clear_current()
def setUp(self):
self.io_loop = None # type: typing.Optional[IOLoop]
IOLoop.clear_current()
class CompatibilityTests(unittest.TestCase):
def setUp(self):
self.saved_signals = save_signal_handlers()
self.io_loop = IOLoop()
self.io_loop.make_current()
self.reactor = TornadoReactor(self.io_loop)
def tearDown(self):
self.reactor.disconnectAll()
self.io_loop.clear_current()
self.io_loop.close(all_fds=True)
restore_signal_handlers(self.saved_signals)
def start_twisted_server(self):
class HelloResource(Resource):
isLeaf = True
def render_GET(self, request):
return "Hello from twisted!"
site = Site(HelloResource())
port = self.reactor.listenTCP(0, site, interface='127.0.0.1')
self.twisted_port = port.getHost().port
def start_tornado_server(self):
class HelloHandler(RequestHandler):
def get(self):
self.write("Hello from tornado!")
app = Application([('/', HelloHandler)],
log_function=lambda x: None)
server = HTTPServer(app, io_loop=self.io_loop)
sock, self.tornado_port = bind_unused_port()
server.add_sockets([sock])
def run_ioloop(self):
self.stop_loop = self.io_loop.stop
self.io_loop.start()
self.reactor.fireSystemEvent('shutdown')
def run_reactor(self):
self.stop_loop = self.reactor.stop
self.stop = self.reactor.stop
self.reactor.run()
def tornado_fetch(self, url, runner):
responses = []
client = AsyncHTTPClient(self.io_loop)
def callback(response):
responses.append(response)
self.stop_loop()
client.fetch(url, callback=callback)
runner()
self.assertEqual(len(responses), 1)
responses[0].rethrow()
return responses[0]
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request('GET', url)
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(ignored):
self.stop_loop()
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return ''.join(chunks)
def testTwistedServerTornadoClientIOLoop(self):
self.start_twisted_server()
response = self.tornado_fetch(
'http://localhost:%d' % self.twisted_port, self.run_ioloop)
self.assertEqual(response.body, 'Hello from twisted!')
def testTwistedServerTornadoClientReactor(self):
self.start_twisted_server()
response = self.tornado_fetch(
'http://localhost:%d' % self.twisted_port, self.run_reactor)
self.assertEqual(response.body, 'Hello from twisted!')
def testTornadoServerTwistedClientIOLoop(self):
self.start_tornado_server()
response = self.twisted_fetch(
'http://localhost:%d' % self.tornado_port, self.run_ioloop)
self.assertEqual(response, 'Hello from tornado!')
def testTornadoServerTwistedClientReactor(self):
self.start_tornado_server()
response = self.twisted_fetch(
'http://localhost:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, 'Hello from tornado!')
def tearDown(self):
super(TestIOLoop, self).tearDown()
BaseIOLoop.clear_current()
BaseIOLoop.clear_instance()
def setUp(self):
setup_with_context_manager(self, ignore_deprecation())
self.io_loop = None # type: typing.Optional[IOLoop]
IOLoop.clear_current()
def __init__(self,
max_workers: Optional[int] = 2,
storage_uri: Optional[str] = None,
storage_project_name: str = "temporary_snowflake",
max_active_services: int = 20,
logging: Union[bool, str] = False,
start_server: bool = True,
reset_database: bool = False):
"""A temporary FractalServer that can be used to run complex workflows or try new computations.
! Warning ! All data is lost when the server is shutdown.
Parameters
----------
max_workers : Optional[int], optional
The maximum number of ProcessPoolExecutor to spin up.
storage_uri : Optional[str], optional
A database URI to connect to, otherwise builds a default instance in a
temporary directory
storage_project_name : str, optional
The database name
max_active_services : int, optional
The maximum number of active services
logging : Union[bool, str], optional
If True, prints logging information to stdout. If False, hides all logging output. If a filename string is provided the logging will be
written to this file.
start_server : bool, optional
Starts the background asyncio loop or not.
reset_database : bool, optional
Resets the database or not if a storage_uri is provided
"""
# Startup a MongoDB in background thread and in custom folder.
if storage_uri is None:
self._storage = TemporaryPostgres(
database_name=storage_project_name)
self._storage_uri = self._storage.database_uri(safe=False,
database="")
else:
self._storage = None
self._storage_uri = storage_uri
if reset_database:
socket = storage_socket_factory(
self._storage_uri, project_name=storage_project_name)
socket._clear_db(socket._project_name)
del socket
# Boot workers if needed
self.queue_socket = None
if max_workers:
self.queue_socket = ProcessPoolExecutor(max_workers=max_workers)
# Add the loop to a background thread and init the server
self.aioloop = asyncio.new_event_loop()
asyncio.set_event_loop(self.aioloop)
IOLoop.clear_instance()
IOLoop.clear_current()
loop = IOLoop()
self.loop = loop
self.loop_thread = ThreadPoolExecutor(max_workers=2)
if logging is False:
self.logfile = tempfile.NamedTemporaryFile()
log_prefix = self.logfile.name
elif logging is True:
self.logfile = None
log_prefix = None
elif isinstance(logging, str):
self.logfile = logging
log_prefix = self.logfile
else:
raise KeyError(f"Logfile type not recognized {type(logging)}.")
super().__init__(name="QCFractal Snowflake Instance",
port=find_port(),
loop=self.loop,
storage_uri=self._storage_uri,
storage_project_name=storage_project_name,
ssl_options=False,
max_active_services=max_active_services,
queue_socket=self.queue_socket,
logfile_prefix=log_prefix,
service_frequency=2,
query_limit=int(1.e6))
if self._storage:
self.logger.warning(
"Warning! This is a temporary instance, data will be lost upon shutdown. "
"For information about how to set up a permanent QCFractal instance, see "
"http://docs.qcarchive.molssi.org/projects/qcfractal/en/latest/setup_quickstart.html"
)
if start_server:
self.start(start_loop=False)
self.loop_future = self.loop_thread.submit(self.loop.start)
self._active = True
# We need to call before threadings cleanup
atexit.register(self.stop)
class CompatibilityTests(unittest.TestCase):
def setUp(self):
self.saved_signals = save_signal_handlers()
self.io_loop = IOLoop()
self.io_loop.make_current()
self.reactor = AsyncioSelectorReactor()
def tearDown(self):
self.reactor.disconnectAll()
self.io_loop.clear_current()
self.io_loop.close(all_fds=True)
restore_signal_handlers(self.saved_signals)
def start_twisted_server(self):
class HelloResource(Resource):
isLeaf = True
def render_GET(self, request):
return b"Hello from twisted!"
site = Site(HelloResource())
port = self.reactor.listenTCP(0, site, interface='127.0.0.1')
self.twisted_port = port.getHost().port
def start_tornado_server(self):
class HelloHandler(RequestHandler):
def get(self):
self.write("Hello from tornado!")
app = Application([('/', HelloHandler)], log_function=lambda x: None)
server = HTTPServer(app)
sock, self.tornado_port = bind_unused_port()
server.add_sockets([sock])
def run_reactor(self):
# In theory, we can run the event loop through Tornado,
# Twisted, or asyncio interfaces. However, since we're trying
# to avoid installing anything as the global event loop, only
# the twisted interface gets everything wired up correectly
# without extra hacks. This method is a part of a
# no-longer-used generalization that allowed us to test
# different combinations.
self.stop_loop = self.reactor.stop
self.stop = self.reactor.stop
self.reactor.run()
def tornado_fetch(self, url, runner):
client = AsyncHTTPClient()
fut = client.fetch(url)
fut.add_done_callback(lambda f: self.stop_loop())
runner()
return fut.result()
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request(b'GET', utf8(url))
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(failure):
if hasattr(self, 'stop_loop'):
self.stop_loop()
elif failure is not None:
# loop hasn't been initialized yet; try our best to
# get an error message out. (the runner() interaction
# should probably be refactored).
try:
failure.raiseException()
except:
logging.error('exception before starting loop',
exc_info=True)
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return b''.join(chunks)
def twisted_coroutine_fetch(self, url, runner):
body = [None]
@gen.coroutine
def f():
# This is simpler than the non-coroutine version, but it cheats
# by reading the body in one blob instead of streaming it with
# a Protocol.
client = Agent(self.reactor)
response = yield client.request(b'GET', utf8(url))
with warnings.catch_warnings():
# readBody has a buggy DeprecationWarning in Twisted 15.0:
# https://twistedmatrix.com/trac/changeset/43379
warnings.simplefilter('ignore', category=DeprecationWarning)
body[0] = yield readBody(response)
self.stop_loop()
self.io_loop.add_callback(f)
runner()
return body[0]
def testTwistedServerTornadoClientReactor(self):
self.start_twisted_server()
response = self.tornado_fetch(
'http://127.0.0.1:%d' % self.twisted_port, self.run_reactor)
self.assertEqual(response.body, b'Hello from twisted!')
def testTornadoServerTwistedClientReactor(self):
self.start_tornado_server()
response = self.twisted_fetch(
'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, b'Hello from tornado!')
def testTornadoServerTwistedCoroutineClientReactor(self):
self.start_tornado_server()
response = self.twisted_coroutine_fetch(
'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, b'Hello from tornado!')
def __init__(self):
# always use a new ioloop
IOLoop.clear_current()
IOLoop(make_current=True)
super(_TestReactor, self).__init__()
IOLoop.clear_current()
class CompatibilityTests(unittest.TestCase):
def setUp(self):
self.saved_signals = save_signal_handlers()
self.io_loop = IOLoop()
self.io_loop.make_current()
self.reactor = AsyncioSelectorReactor()
def tearDown(self):
self.reactor.disconnectAll()
self.io_loop.clear_current()
self.io_loop.close(all_fds=True)
restore_signal_handlers(self.saved_signals)
def start_twisted_server(self):
class HelloResource(Resource):
isLeaf = True
def render_GET(self, request):
return b"Hello from twisted!"
site = Site(HelloResource())
port = self.reactor.listenTCP(0, site, interface='127.0.0.1')
self.twisted_port = port.getHost().port
def start_tornado_server(self):
class HelloHandler(RequestHandler):
def get(self):
self.write("Hello from tornado!")
app = Application([('/', HelloHandler)],
log_function=lambda x: None)
server = HTTPServer(app)
sock, self.tornado_port = bind_unused_port()
server.add_sockets([sock])
def run_reactor(self):
# In theory, we can run the event loop through Tornado,
# Twisted, or asyncio interfaces. However, since we're trying
# to avoid installing anything as the global event loop, only
# the twisted interface gets everything wired up correectly
# without extra hacks. This method is a part of a
# no-longer-used generalization that allowed us to test
# different combinations.
self.stop_loop = self.reactor.stop
self.stop = self.reactor.stop
self.reactor.run()
def tornado_fetch(self, url, runner):
client = AsyncHTTPClient()
fut = client.fetch(url)
fut.add_done_callback(lambda f: self.stop_loop())
runner()
return fut.result()
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request(b'GET', utf8(url))
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(failure):
if hasattr(self, 'stop_loop'):
self.stop_loop()
elif failure is not None:
# loop hasn't been initialized yet; try our best to
# get an error message out. (the runner() interaction
# should probably be refactored).
try:
failure.raiseException()
except:
logging.error('exception before starting loop', exc_info=True)
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return b''.join(chunks)
def twisted_coroutine_fetch(self, url, runner):
body = [None]
@gen.coroutine
def f():
# This is simpler than the non-coroutine version, but it cheats
# by reading the body in one blob instead of streaming it with
# a Protocol.
client = Agent(self.reactor)
response = yield client.request(b'GET', utf8(url))
with warnings.catch_warnings():
# readBody has a buggy DeprecationWarning in Twisted 15.0:
# https://twistedmatrix.com/trac/changeset/43379
warnings.simplefilter('ignore', category=DeprecationWarning)
body[0] = yield readBody(response)
self.stop_loop()
self.io_loop.add_callback(f)
runner()
return body[0]
def testTwistedServerTornadoClientReactor(self):
self.start_twisted_server()
response = self.tornado_fetch(
'http://127.0.0.1:%d' % self.twisted_port, self.run_reactor)
self.assertEqual(response.body, b'Hello from twisted!')
def testTornadoServerTwistedClientReactor(self):
self.start_tornado_server()
response = self.twisted_fetch(
'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, b'Hello from tornado!')
def testTornadoServerTwistedCoroutineClientReactor(self):
self.start_tornado_server()
response = self.twisted_coroutine_fetch(
'http://127.0.0.1:%d' % self.tornado_port, self.run_reactor)
self.assertEqual(response, b'Hello from tornado!')
def setUp(self):
self.io_loop = None
IOLoop.clear_current()
class CompatibilityTests(unittest.TestCase):
def setUp(self):
self.saved_signals = save_signal_handlers()
self.io_loop = IOLoop()
self.io_loop.make_current()
self.reactor = TornadoReactor(self.io_loop)
def tearDown(self):
self.reactor.disconnectAll()
self.io_loop.clear_current()
self.io_loop.close(all_fds=True)
restore_signal_handlers(self.saved_signals)
def start_twisted_server(self):
class HelloResource(Resource):
isLeaf = True
def render_GET(self, request):
return "Hello from twisted!"
site = Site(HelloResource())
port = self.reactor.listenTCP(0, site, interface="127.0.0.1")
self.twisted_port = port.getHost().port
def start_tornado_server(self):
class HelloHandler(RequestHandler):
def get(self):
self.write("Hello from tornado!")
app = Application([("/", HelloHandler)], log_function=lambda x: None)
server = HTTPServer(app, io_loop=self.io_loop)
sock, self.tornado_port = bind_unused_port()
server.add_sockets([sock])
def run_ioloop(self):
self.stop_loop = self.io_loop.stop
self.io_loop.start()
self.reactor.fireSystemEvent("shutdown")
def run_reactor(self):
self.stop_loop = self.reactor.stop
self.stop = self.reactor.stop
self.reactor.run()
def tornado_fetch(self, url, runner):
responses = []
client = AsyncHTTPClient(self.io_loop)
def callback(response):
responses.append(response)
self.stop_loop()
client.fetch(url, callback=callback)
runner()
self.assertEqual(len(responses), 1)
responses[0].rethrow()
return responses[0]
def twisted_fetch(self, url, runner):
# http://twistedmatrix.com/documents/current/web/howto/client.html
chunks = []
client = Agent(self.reactor)
d = client.request(b"GET", utf8(url))
class Accumulator(Protocol):
def __init__(self, finished):
self.finished = finished
def dataReceived(self, data):
chunks.append(data)
def connectionLost(self, reason):
self.finished.callback(None)
def callback(response):
finished = Deferred()
response.deliverBody(Accumulator(finished))
return finished
d.addCallback(callback)
def shutdown(failure):
if hasattr(self, "stop_loop"):
self.stop_loop()
elif failure is not None:
# loop hasn't been initialized yet; try our best to
# get an error message out. (the runner() interaction
# should probably be refactored).
try:
failure.raiseException()
except:
logging.error("exception before starting loop", exc_info=True)
d.addBoth(shutdown)
runner()
self.assertTrue(chunks)
return "".join(chunks)
def twisted_coroutine_fetch(self, url, runner):
body = [None]
@gen.coroutine
def f():
# This is simpler than the non-coroutine version, but it cheats
# by reading the body in one blob instead of streaming it with
# a Protocol.
client = Agent(self.reactor)
response = yield client.request(b"GET", utf8(url))
with warnings.catch_warnings():
# readBody has a buggy DeprecationWarning in Twisted 15.0:
# https://twistedmatrix.com/trac/changeset/43379
warnings.simplefilter("ignore", category=DeprecationWarning)
body[0] = yield readBody(response)
self.stop_loop()
self.io_loop.add_callback(f)
runner()
return body[0]
def testTwistedServerTornadoClientIOLoop(self):
self.start_twisted_server()
response = self.tornado_fetch("http://127.0.0.1:%d" % self.twisted_port, self.run_ioloop)
self.assertEqual(response.body, "Hello from twisted!")
def testTwistedServerTornadoClientReactor(self):
self.start_twisted_server()
response = self.tornado_fetch("http://127.0.0.1:%d" % self.twisted_port, self.run_reactor)
self.assertEqual(response.body, "Hello from twisted!")
def testTornadoServerTwistedClientIOLoop(self):
self.start_tornado_server()
response = self.twisted_fetch("http://127.0.0.1:%d" % self.tornado_port, self.run_ioloop)
self.assertEqual(response, "Hello from tornado!")
def testTornadoServerTwistedClientReactor(self):
self.start_tornado_server()
response = self.twisted_fetch("http://127.0.0.1:%d" % self.tornado_port, self.run_reactor)
self.assertEqual(response, "Hello from tornado!")
@skipIfPy26
def testTornadoServerTwistedCoroutineClientIOLoop(self):
self.start_tornado_server()
response = self.twisted_coroutine_fetch("http://127.0.0.1:%d" % self.tornado_port, self.run_ioloop)
self.assertEqual(response, "Hello from tornado!")
def __init__(self):
# always use a new ioloop
IOLoop.clear_current()
IOLoop(make_current=True)
super(_TestReactor, self).__init__()
IOLoop.clear_current()
