class ReactorTestCase(unittest.TestCase): def setUp(self): self._saved_signals = save_signal_handlers() self._io_loop = IOLoop() self._reactor = TornadoReactor(self._io_loop) def tearDown(self): self._io_loop.close(all_fds=True) restore_signal_handlers(self._saved_signals)
def configure_ioloop(): kwargs = {} if options.ioloop_time_monotonic: from webalchemy.tornado.platform.auto import monotonic_time if monotonic_time is None: raise RuntimeError("monotonic clock not found") kwargs['time_func'] = monotonic_time if options.ioloop or kwargs: IOLoop.configure(options.ioloop, **kwargs)
def test_add_callback_from_signal_other_thread(self): # Very crude test, just to make sure that we cover this case. # This also happens to be the first test where we run an IOLoop in # a non-main thread. other_ioloop = IOLoop() thread = threading.Thread(target=other_ioloop.start) thread.start() other_ioloop.add_callback_from_signal(other_ioloop.stop) thread.join() other_ioloop.close()
def tearDown(self): self.io_loop.clear_current() if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()): # Try to clean up any file descriptors left open in the ioloop. # This avoids leaks, especially when tests are run repeatedly # in the same process with autoreload (because curl does not # set FD_CLOEXEC on its file descriptors) self.io_loop.close(all_fds=True) super(AsyncTestCase, self).tearDown() # In case an exception escaped or the StackContext caught an exception # when there wasn't a wait() to re-raise it, do so here. # This is our last chance to raise an exception in a way that the # unittest machinery understands. self.__rethrow()
def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None): """Client-side websocket support. Takes a url and returns a Future whose result is a `WebSocketClientConnection`. .. versionchanged:: 3.2 Also accepts ``HTTPRequest`` objects in place of urls. """ if io_loop is None: io_loop = IOLoop.current() if isinstance(url, httpclient.HTTPRequest): assert connect_timeout is None request = url # Copy and convert the headers dict/object (see comments in # AsyncHTTPClient.fetch) request.headers = httputil.HTTPHeaders(request.headers) else: request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) request = httpclient._RequestProxy( request, httpclient.HTTPRequest._DEFAULTS) conn = WebSocketClientConnection(io_loop, request) if callback is not None: io_loop.add_future(conn.connect_future, callback) return conn.connect_future
def add_accept_handler(sock, callback, io_loop=None): """Adds an `.IOLoop` event handler to accept new connections on ``sock``. When a connection is accepted, ``callback(connection, address)`` will be run (``connection`` is a socket object, and ``address`` is the address of the other end of the connection). Note that this signature is different from the ``callback(fd, events)`` signature used for `.IOLoop` handlers. """ if io_loop is None: io_loop = IOLoop.current() def accept_handler(fd, events): while True: try: connection, address = sock.accept() except socket.error as e: # EWOULDBLOCK and EAGAIN indicate we have accepted every # connection that is available. if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): return # ECONNABORTED indicates that there was a connection # but it was closed while still in the accept queue. # (observed on FreeBSD). if e.args[0] == errno.ECONNABORTED: continue raise callback(connection, address) io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ)
def test_subprocess(self): if IOLoop.configured_class().__name__.endswith("LayeredTwistedIOLoop"): # This test fails non-deterministically with LayeredTwistedIOLoop. # (the read_until('\n') returns '\n' instead of 'hello\n') # This probably indicates a problem with either TornadoReactor # or TwistedIOLoop, but I haven't been able to track it down # and for now this is just causing spurious travis-ci failures. raise unittest.SkipTest("Subprocess tests not compatible with " "LayeredTwistedIOLoop") subproc = Subprocess( [sys.executable, "-u", "-i"], stdin=Subprocess.STREAM, stdout=Subprocess.STREAM, stderr=subprocess.STDOUT, io_loop=self.io_loop, ) self.addCleanup(lambda: os.kill(subproc.pid, signal.SIGTERM)) subproc.stdout.read_until(b">>> ", self.stop) self.wait() subproc.stdin.write(b"print('hello')\n") subproc.stdout.read_until(b"\n", self.stop) data = self.wait() self.assertEqual(data, b"hello\n") subproc.stdout.read_until(b">>> ", self.stop) self.wait() subproc.stdin.write(b"raise SystemExit\n") subproc.stdout.read_until_close(self.stop) data = self.wait() self.assertEqual(data, b"")
def initialize(self, io_loop=None, executor=None, close_executor=True): self.io_loop = io_loop or IOLoop.current() if executor is not None: self.executor = executor self.close_executor = close_executor else: self.executor = dummy_executor self.close_executor = False
class HTTPClient(object): """A blocking HTTP client. This interface is provided for convenience and testing; most applications that are running an IOLoop will want to use `AsyncHTTPClient` instead. Typical usage looks like this:: http_client = httpclient.HTTPClient() try: response = http_client.fetch("http://www.google.com/") print response.body except httpclient.HTTPError as e: print "Error:", e http_client.close() """ def __init__(self, async_client_class=None, **kwargs): self._io_loop = IOLoop() if async_client_class is None: async_client_class = AsyncHTTPClient self._async_client = async_client_class(self._io_loop, **kwargs) self._closed = False def __del__(self): self.close() def close(self): """Closes the HTTPClient, freeing any resources used.""" if not self._closed: self._async_client.close() self._io_loop.close() self._closed = True def fetch(self, request, **kwargs): """Executes a request, returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` If an error occurs during the fetch, we raise an `HTTPError`. """ response = self._io_loop.run_sync(functools.partial( self._async_client.fetch, request, **kwargs)) response.rethrow() return response
def __new__(cls, io_loop=None, force_instance=False, **kwargs): io_loop = io_loop or IOLoop.current() if io_loop in cls._async_clients() and not force_instance: return cls._async_clients()[io_loop] instance = super(AsyncHTTPClient, cls).__new__(cls, io_loop=io_loop, **kwargs) if not force_instance: cls._async_clients()[io_loop] = instance return instance
def save_signal_handlers(): saved = {} for sig in [signal.SIGINT, signal.SIGTERM, signal.SIGCHLD]: saved[sig] = signal.getsignal(sig) if "twisted" in repr(saved): if not issubclass(IOLoop.configured_class(), TwistedIOLoop): # when the global ioloop is twisted, we expect the signal # handlers to be installed. Otherwise, it means we're not # cleaning up after twisted properly. raise Exception("twisted signal handlers already installed") return saved
def test_add_callback_while_closing(self): # Issue #635: add_callback() should raise a clean exception # if called while another thread is closing the IOLoop. closing = threading.Event() def target(): other_ioloop.add_callback(other_ioloop.stop) other_ioloop.start() closing.set() other_ioloop.close(all_fds=True) other_ioloop = IOLoop() thread = threading.Thread(target=target) thread.start() closing.wait() for i in range(1000): try: other_ioloop.add_callback(lambda: None) except RuntimeError as e: self.assertEqual("IOLoop is closing", str(e)) break
def initialize(self, io_loop=None): self.io_loop = io_loop or IOLoop.current() # partial copy of twisted.names.client.createResolver, which doesn't # allow for a reactor to be passed in. self.reactor = webalchemy.tornado.platform.twisted.TornadoReactor(io_loop) host_resolver = twisted.names.hosts.Resolver('/etc/hosts') cache_resolver = twisted.names.cache.CacheResolver(reactor=self.reactor) real_resolver = twisted.names.client.Resolver('/etc/resolv.conf', reactor=self.reactor) self.resolver = twisted.names.resolve.ResolverChain( [host_resolver, cache_resolver, real_resolver])
class TestIOLoopCurrent(unittest.TestCase): def setUp(self): self.io_loop = IOLoop() def tearDown(self): self.io_loop.close() def test_current(self): def f(): self.current_io_loop = IOLoop.current() self.io_loop.stop() self.io_loop.add_callback(f) self.io_loop.start() self.assertIs(self.current_io_loop, self.io_loop)
def setUp(self): if IOLoop.configured_class().__name__ in ('TwistedIOLoop', 'AsyncIOMainLoop'): # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. # AsyncIOMainLoop doesn't work with the default policy # (although it could with some tweaks to this test and a # policy that created loops for non-main threads). raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop or ' 'AsyncIOMainLoop') self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient()
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() if 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) def handle_exception(typ, value, tb): try: if runner is not None and runner.handle_exception(typ, value, tb): return True except Exception: typ, value, tb = sys.exc_info() future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception) as deactivate: try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: deactivate() future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): def final_callback(value): deactivate() future.set_result(value) runner = Runner(result, final_callback) runner.run() return future deactivate() future.set_result(result) return future
def add_sockets(self, sockets): """Makes this server start accepting connections on the given sockets. The ``sockets`` parameter is a list of socket objects such as those returned by `~webalchemy.tornado.netutil.bind_sockets`. `add_sockets` is typically used in combination with that method and `webalchemy.tornado.process.fork_processes` to provide greater control over the initialization of a multi-process server. """ if self.io_loop is None: self.io_loop = IOLoop.current() for sock in sockets: self._sockets[sock.fileno()] = sock add_accept_handler(sock, self._handle_connection, io_loop=self.io_loop)
class SyncHTTPClientTest(unittest.TestCase): def setUp(self): if IOLoop.configured_class().__name__ in ('TwistedIOLoop', 'AsyncIOMainLoop'): # TwistedIOLoop only supports the global reactor, so we can't have # separate IOLoops for client and server threads. # AsyncIOMainLoop doesn't work with the default policy # (although it could with some tweaks to this test and a # policy that created loops for non-main threads). raise unittest.SkipTest( 'Sync HTTPClient not compatible with TwistedIOLoop or ' 'AsyncIOMainLoop') self.server_ioloop = IOLoop() sock, self.port = bind_unused_port() app = Application([('/', HelloWorldHandler)]) self.server = HTTPServer(app, io_loop=self.server_ioloop) self.server.add_socket(sock) self.server_thread = threading.Thread(target=self.server_ioloop.start) self.server_thread.start() self.http_client = HTTPClient() def tearDown(self): def stop_server(): self.server.stop() self.server_ioloop.stop() self.server_ioloop.add_callback(stop_server) self.server_thread.join() self.http_client.close() self.server_ioloop.close(all_fds=True) def get_url(self, path): return 'http://localhost:%d%s' % (self.port, path) def test_sync_client(self): response = self.http_client.fetch(self.get_url('/')) self.assertEqual(b'Hello world!', response.body) def test_sync_client_error(self): # Synchronous HTTPClient raises errors directly; no need for # response.rethrow() with self.assertRaises(HTTPError) as assertion: self.http_client.fetch(self.get_url('/notfound')) self.assertEqual(assertion.exception.code, 404)
class TestIOLoopRunSync(unittest.TestCase): def setUp(self): self.io_loop = IOLoop() def tearDown(self): self.io_loop.close() def test_sync_result(self): self.assertEqual(self.io_loop.run_sync(lambda: 42), 42) def test_sync_exception(self): with self.assertRaises(ZeroDivisionError): self.io_loop.run_sync(lambda: 1 / 0) def test_async_result(self): @gen.coroutine def f(): yield gen.Task(self.io_loop.add_callback) raise gen.Return(42) self.assertEqual(self.io_loop.run_sync(f), 42) def test_async_exception(self): @gen.coroutine def f(): yield gen.Task(self.io_loop.add_callback) 1 / 0 with self.assertRaises(ZeroDivisionError): self.io_loop.run_sync(f) def test_current(self): def f(): self.assertIs(IOLoop.current(), self.io_loop) self.io_loop.run_sync(f) def test_timeout(self): @gen.coroutine def f(): yield gen.Task(self.io_loop.add_timeout, self.io_loop.time() + 1) self.assertRaises(TimeoutError, self.io_loop.run_sync, f, timeout=0.01)
def setUp(self): self._saved_signals = save_signal_handlers() self._io_loop = IOLoop() self._reactor = TornadoReactor(self._io_loop)
def setUp(self): self.saved_signals = save_signal_handlers() self.io_loop = IOLoop() self.io_loop.make_current() self.reactor = TornadoReactor(self.io_loop)
if have_twisted: # Import and run as much of twisted's test suite as possible. # This is unfortunately rather dependent on implementation details, # but there doesn't appear to be a clean all-in-one conformance test # suite for reactors. # # This is a list of all test suites using the ReactorBuilder # available in Twisted 11.0.0 and 11.1.0 (and a blacklist of # specific test methods to be disabled). twisted_tests = { 'twisted.internet.test.test_core.ObjectModelIntegrationTest': [], 'twisted.internet.test.test_core.SystemEventTestsBuilder': [ 'test_iterate', # deliberately not supported 'test_runAfterCrash', # fails because TwistedIOLoop uses the global reactor ] if issubclass(IOLoop.configured_class(), TwistedIOLoop) else [ 'test_iterate', # deliberately not supported ], 'twisted.internet.test.test_fdset.ReactorFDSetTestsBuilder': [ "test_lostFileDescriptor", # incompatible with epoll and kqueue ], 'twisted.internet.test.test_process.ProcessTestsBuilder': [ # Only work as root. Twisted's "skip" functionality works # with py27+, but not unittest2 on py26. 'test_changeGID', 'test_changeUID', ], # Process tests appear to work on OSX 10.7, but not 10.6 #'twisted.internet.test.test_process.PTYProcessTestsBuilder': [ # 'test_systemCallUninterruptedByChildExit', # ],
class CompatibilityTests(unittest.TestCase): def setUp(self): self.saved_signals = save_signal_handlers() self.io_loop = IOLoop() self.io_loop.make_current() self.reactor = TornadoReactor(self.io_loop) def tearDown(self): self.reactor.disconnectAll() self.io_loop.clear_current() self.io_loop.close(all_fds=True) restore_signal_handlers(self.saved_signals) def start_twisted_server(self): class HelloResource(Resource): isLeaf = True def render_GET(self, request): return "Hello from twisted!" site = Site(HelloResource()) port = self.reactor.listenTCP(0, site, interface='127.0.0.1') self.twisted_port = port.getHost().port def start_tornado_server(self): class HelloHandler(RequestHandler): def get(self): self.write("Hello from webalchemy.tornado!") app = Application([('/', HelloHandler)], log_function=lambda x: None) server = HTTPServer(app, io_loop=self.io_loop) sock, self.tornado_port = bind_unused_port() server.add_sockets([sock]) def run_ioloop(self): self.stop_loop = self.io_loop.stop self.io_loop.start() self.reactor.fireSystemEvent('shutdown') def run_reactor(self): self.stop_loop = self.reactor.stop self.stop = self.reactor.stop self.reactor.run() def tornado_fetch(self, url, runner): responses = [] client = AsyncHTTPClient(self.io_loop) def callback(response): responses.append(response) self.stop_loop() client.fetch(url, callback=callback) runner() self.assertEqual(len(responses), 1) responses[0].rethrow() return responses[0] def twisted_fetch(self, url, runner): # http://twistedmatrix.com/documents/current/web/howto/client.html chunks = [] client = Agent(self.reactor) d = client.request('GET', url) class Accumulator(Protocol): def __init__(self, finished): self.finished = finished def dataReceived(self, data): chunks.append(data) def connectionLost(self, reason): self.finished.callback(None) def callback(response): finished = Deferred() response.deliverBody(Accumulator(finished)) return finished d.addCallback(callback) def shutdown(ignored): self.stop_loop() d.addBoth(shutdown) runner() self.assertTrue(chunks) return ''.join(chunks) def testTwistedServerTornadoClientIOLoop(self): self.start_twisted_server() response = self.tornado_fetch( 'http://localhost:%d' % self.twisted_port, self.run_ioloop) self.assertEqual(response.body, 'Hello from twisted!') def testTwistedServerTornadoClientReactor(self): self.start_twisted_server() response = self.tornado_fetch( 'http://localhost:%d' % self.twisted_port, self.run_reactor) self.assertEqual(response.body, 'Hello from twisted!') def testTornadoServerTwistedClientIOLoop(self): self.start_tornado_server() response = self.twisted_fetch( 'http://localhost:%d' % self.tornado_port, self.run_ioloop) self.assertEqual(response, 'Hello from webalchemy.tornado!') def testTornadoServerTwistedClientReactor(self): self.start_tornado_server() response = self.twisted_fetch( 'http://localhost:%d' % self.tornado_port, self.run_reactor) self.assertEqual(response, 'Hello from webalchemy.tornado!')
def f(): self.current_io_loop = IOLoop.current() self.io_loop.stop()
def __init__(self, future, io_loop=None): self.future = future self.io_loop = io_loop or IOLoop.current()
def initialize(self, io_loop=None): self.io_loop = io_loop or IOLoop.current() self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb) self.fds = {}
def skip_if_twisted(): if IOLoop.configured_class().__name__.endswith(("TwistedIOLoop", "AsyncIOMainLoop")): raise unittest.SkipTest("Process tests not compatible with " "TwistedIOLoop or AsyncIOMainLoop")
def test_multi_process(self): # This test can't work on twisted because we use the global reactor # and have no way to get it back into a sane state after the fork. skip_if_twisted() with ExpectLog(gen_log, "(Starting .* processes|child .* exited|uncaught exception)"): self.assertFalse(IOLoop.initialized()) sock, port = bind_unused_port() def get_url(path): return "http://127.0.0.1:%d%s" % (port, path) # ensure that none of these processes live too long signal.alarm(5) # master process try: id = fork_processes(3, max_restarts=3) self.assertTrue(id is not None) signal.alarm(5) # child processes except SystemExit as e: # if we exit cleanly from fork_processes, all the child processes # finished with status 0 self.assertEqual(e.code, 0) self.assertTrue(task_id() is None) sock.close() return try: if id in (0, 1): self.assertEqual(id, task_id()) server = HTTPServer(self.get_app()) server.add_sockets([sock]) IOLoop.instance().start() elif id == 2: self.assertEqual(id, task_id()) sock.close() # Always use SimpleAsyncHTTPClient here; the curl # version appears to get confused sometimes if the # connection gets closed before it's had a chance to # switch from writing mode to reading mode. client = HTTPClient(SimpleAsyncHTTPClient) def fetch(url, fail_ok=False): try: return client.fetch(get_url(url)) except HTTPError as e: if not (fail_ok and e.code == 599): raise # Make two processes exit abnormally fetch("/?exit=2", fail_ok=True) fetch("/?exit=3", fail_ok=True) # They've been restarted, so a new fetch will work int(fetch("/").body) # Now the same with signals # Disabled because on the mac a process dying with a signal # can trigger an "Application exited abnormally; send error # report to Apple?" prompt. # fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True) # fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True) # int(fetch("/").body) # Now kill them normally so they won't be restarted fetch("/?exit=0", fail_ok=True) # One process left; watch it's pid change pid = int(fetch("/").body) fetch("/?exit=4", fail_ok=True) pid2 = int(fetch("/").body) self.assertNotEqual(pid, pid2) # Kill the last one so we shut down cleanly fetch("/?exit=0", fail_ok=True) os._exit(0) except Exception: logging.error("exception in child process %d", id, exc_info=True) raise
def f(): self.assertIs(IOLoop.current(), self.io_loop)
def setUp(self): self.io_loop = IOLoop()