Beispiel #1
0
    def read_until_close(self, callback, streaming_callback=None):
        """Reads all data from the socket until it is closed.

        If a ``streaming_callback`` is given, it will be called with chunks
        of data as they become available, and the argument to the final
        ``callback`` will be empty.  Otherwise, the ``callback`` gets the
        data as an argument.

        Subject to ``max_buffer_size`` limit from `IOStream` constructor if
        a ``streaming_callback`` is not used.
        """
        self._set_read_callback(callback)
        self._streaming_callback = stack_context.wrap(streaming_callback)
        if self.closed():
            if self._streaming_callback is not None:
                self._run_callback(self._streaming_callback,
                                   self._consume(self._read_buffer_size))
            self._run_callback(self._read_callback,
                               self._consume(self._read_buffer_size))
            self._streaming_callback = None
            self._read_callback = None
            return
        self._read_until_close = True
        self._streaming_callback = stack_context.wrap(streaming_callback)
        self._try_inline_read()
Beispiel #2
0
 def add_callback(self, callback, *args, **kwargs):
     if self.closing:
         raise RuntimeError("IOLoop is closing")
     if kwargs:
         self.asyncio_loop.call_soon_threadsafe(
             functools.partial(self._run_callback, stack_context.wrap(callback), *args, **kwargs)
         )
     else:
         self.asyncio_loop.call_soon_threadsafe(self._run_callback, stack_context.wrap(callback), *args)
Beispiel #3
0
    def write(self, data, callback=None):
        """Write the given data to this stream.

        If ``callback`` is given, we call it when all of the buffered write
        data has been successfully written to the stream. If there was
        previously buffered write data and an old write callback, that
        callback is simply overwritten with this new callback.
        """
        assert isinstance(data, bytes_type)
        self._check_closed()
        # We use bool(_write_buffer) as a proxy for write_buffer_size>0,
        # so never put empty strings in the buffer.
        if data:
            # Break up large contiguous strings before inserting them in the
            # write buffer, so we don't have to recopy the entire thing
            # as we slice off pieces to send to the socket.
            WRITE_BUFFER_CHUNK_SIZE = 128 * 1024
            if len(data) > WRITE_BUFFER_CHUNK_SIZE:
                for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
                    self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
            else:
                self._write_buffer.append(data)
        self._write_callback = stack_context.wrap(callback)
        if not self._connecting:
            self._handle_write()
            if self._write_buffer:
                self._add_io_state(self.io_loop.WRITE)
            self._maybe_add_error_listener()
Beispiel #4
0
 def add_timeout(self, deadline, callback):
     if isinstance(deadline, (int, long, float)):
         delay = max(deadline - self.time(), 0)
     elif isinstance(deadline, datetime.timedelta):
         delay = webalchemy.tornado.ioloop._Timeout.timedelta_to_seconds(deadline)
     else:
         raise TypeError("Unsupported deadline %r")
     return self.reactor.callLater(delay, self._run_callback, wrap(callback))
Beispiel #5
0
 def add_timeout(self, deadline, callback):
     if isinstance(deadline, (int, float)):
         delay = max(deadline - self.time(), 0)
     elif isinstance(deadline, datetime.timedelta):
         delay = deadline.total_seconds()
     else:
         raise TypeError("Unsupported deadline %r", deadline)
     return self.asyncio_loop.call_later(delay, self._run_callback, stack_context.wrap(callback))
Beispiel #6
0
    def set_close_callback(self, callback):
        """Sets a callback that will be run when the connection is closed.

        Use this instead of accessing
        `HTTPConnection.stream.set_close_callback
        <.BaseIOStream.set_close_callback>` directly (which was the
        recommended approach prior to Tornado 3.0).
        """
        self._close_callback = stack_context.wrap(callback)
Beispiel #7
0
 def add_handler(self, fd, handler, events):
     if fd in self.handlers:
         raise ValueError("fd %d added twice" % fd)
     self.handlers[fd] = stack_context.wrap(handler)
     if events & IOLoop.READ:
         self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
         self.readers.add(fd)
     if events & IOLoop.WRITE:
         self.asyncio_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
         self.writers.add(fd)
Beispiel #8
0
 def result_callback(self, key):
     def inner(*args, **kwargs):
         if kwargs or len(args) > 1:
             result = Arguments(args, kwargs)
         elif args:
             result = args[0]
         else:
             result = None
         self.set_result(key, result)
     return wrap(inner)
Beispiel #9
0
 def add_handler(self, fd, handler, events):
     if fd in self.fds:
         raise ValueError('fd %d added twice' % fd)
     self.fds[fd] = _FD(fd, wrap(handler))
     if events & webalchemy.tornado.ioloop.IOLoop.READ:
         self.fds[fd].reading = True
         self.reactor.addReader(self.fds[fd])
     if events & webalchemy.tornado.ioloop.IOLoop.WRITE:
         self.fds[fd].writing = True
         self.reactor.addWriter(self.fds[fd])
 def capitalize(self, request_data, callback=None):
     logging.info("capitalize")
     self.request_data = request_data
     self.stream = IOStream(socket.socket(), io_loop=self.io_loop)
     self.stream.connect(('127.0.0.1', self.port),
                         callback=self.handle_connect)
     self.future = Future()
     if callback is not None:
         self.future.add_done_callback(
             stack_context.wrap(lambda future: callback(future.result())))
     return self.future
Beispiel #11
0
    def add_future(self, future, callback):
        """Schedules a callback on the ``IOLoop`` when the given
        `.Future` is finished.

        The callback is invoked with one argument, the
        `.Future`.
        """
        assert isinstance(future, Future)
        callback = stack_context.wrap(callback)
        future.add_done_callback(
            lambda future: self.add_callback(callback, future))
 def make_wrapped_function():
     """Wraps a function in three stack contexts, and returns
     the function along with the deactivation functions.
     """
     # Remove the test's stack context to make sure we can cover
     # the case where the last context is deactivated.
     with NullContext():
         partial = functools.partial
         with StackContext(partial(self.context, 'c0')) as c0:
             with StackContext(partial(self.context, 'c1')) as c1:
                 with StackContext(partial(self.context, 'c2')) as c2:
                     return (wrap(check_contexts), [c0, c1, c2])
Beispiel #13
0
    def read_bytes(self, num_bytes, callback, streaming_callback=None):
        """Run callback when we read the given number of bytes.

        If a ``streaming_callback`` is given, it will be called with chunks
        of data as they become available, and the argument to the final
        ``callback`` will be empty.  Otherwise, the ``callback`` gets
        the data as an argument.
        """
        self._set_read_callback(callback)
        assert isinstance(num_bytes, numbers.Integral)
        self._read_bytes = num_bytes
        self._streaming_callback = stack_context.wrap(streaming_callback)
        self._try_inline_read()
    def __init__(self, io_loop, client, request, release_callback,
                 final_callback, max_buffer_size, resolver):
        self.start_time = io_loop.time()
        self.io_loop = io_loop
        self.client = client
        self.request = request
        self.release_callback = release_callback
        self.final_callback = final_callback
        self.max_buffer_size = max_buffer_size
        self.resolver = resolver
        self.code = None
        self.headers = None
        self.chunks = None
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        with stack_context.ExceptionStackContext(self._handle_exception):
            self.parsed = urlparse.urlsplit(_unicode(self.request.url))
            if self.parsed.scheme not in ("http", "https"):
                raise ValueError("Unsupported url scheme: %s" %
                                 self.request.url)
            # urlsplit results have hostname and port results, but they
            # didn't support ipv6 literals until python 2.7.
            netloc = self.parsed.netloc
            if "@" in netloc:
                userpass, _, netloc = netloc.rpartition("@")
            match = re.match(r'^(.+):(\d+)$', netloc)
            if match:
                host = match.group(1)
                port = int(match.group(2))
            else:
                host = netloc
                port = 443 if self.parsed.scheme == "https" else 80
            if re.match(r'^\[.*\]$', host):
                # raw ipv6 addresses in urls are enclosed in brackets
                host = host[1:-1]
            self.parsed_hostname = host  # save final host for _on_connect

            if request.allow_ipv6:
                af = socket.AF_UNSPEC
            else:
                # We only try the first IP we get from getaddrinfo,
                # so restrict to ipv4 by default.
                af = socket.AF_INET

            timeout = min(self.request.connect_timeout, self.request.request_timeout)
            if timeout:
                self._timeout = self.io_loop.add_timeout(
                    self.start_time + timeout,
                    stack_context.wrap(self._on_timeout))
            self.resolver.resolve(host, port, af, callback=self._on_resolve)
Beispiel #15
0
    def test_pre_wrap(self):
        # A pre-wrapped callback is run in the context in which it was
        # wrapped, not when it was added to the IOLoop.
        def f1():
            self.assertIn('c1', self.active_contexts)
            self.assertNotIn('c2', self.active_contexts)
            self.stop()

        with StackContext(functools.partial(self.context, 'c1')):
            wrapped = wrap(f1)

        with StackContext(functools.partial(self.context, 'c2')):
            self.add_callback(wrapped)

        self.wait()
Beispiel #16
0
 def add_callback(self, callback, *args, **kwargs):
     with self._callback_lock:
         if self._closing:
             raise RuntimeError("IOLoop is closing")
         list_empty = not self._callbacks
         self._callbacks.append(functools.partial(
             stack_context.wrap(callback), *args, **kwargs))
         if list_empty and thread.get_ident() != self._thread_ident:
             # If we're in the IOLoop's thread, we know it's not currently
             # polling.  If we're not, and we added the first callback to an
             # empty list, we may need to wake it up (it may wake up on its
             # own, but an occasional extra wake is harmless).  Waking
             # up a polling IOLoop is relatively expensive, so we try to
             # avoid it when we can.
             self._waker.wake()
Beispiel #17
0
    def fetch(self, request, callback=None, **kwargs):
        """Executes a request, asynchronously returning an `HTTPResponse`.

        The request may be either a string URL or an `HTTPRequest` object.
        If it is a string, we construct an `HTTPRequest` using any additional
        kwargs: ``HTTPRequest(request, **kwargs)``

        This method returns a `.Future` whose result is an
        `HTTPResponse`.  The ``Future`` wil raise an `HTTPError` if
        the request returned a non-200 response code.

        If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
        In the callback interface, `HTTPError` is not automatically raised.
        Instead, you must check the response's ``error`` attribute or
        call its `~HTTPResponse.rethrow` method.
        """
        if not isinstance(request, HTTPRequest):
            request = HTTPRequest(url=request, **kwargs)
        # We may modify this (to add Host, Accept-Encoding, etc),
        # so make sure we don't modify the caller's object.  This is also
        # where normal dicts get converted to HTTPHeaders objects.
        request.headers = httputil.HTTPHeaders(request.headers)
        request = _RequestProxy(request, self.defaults)
        future = TracebackFuture()
        if callback is not None:
            callback = stack_context.wrap(callback)

            def handle_future(future):
                exc = future.exception()
                if isinstance(exc, HTTPError) and exc.response is not None:
                    response = exc.response
                elif exc is not None:
                    response = HTTPResponse(
                        request, 599, error=exc,
                        request_time=time.time() - request.start_time)
                else:
                    response = future.result()
                self.io_loop.add_callback(callback, response)
            future.add_done_callback(handle_future)

        def handle_response(response):
            if response.error:
                future.set_exception(response.error)
            else:
                future.set_result(response)
        self.fetch_impl(request, handle_response)
        return future
Beispiel #18
0
 def add_callback_from_signal(self, callback, *args, **kwargs):
     with stack_context.NullContext():
         if thread.get_ident() != self._thread_ident:
             # if the signal is handled on another thread, we can add
             # it normally (modulo the NullContext)
             self.add_callback(callback, *args, **kwargs)
         else:
             # If we're on the IOLoop's thread, we cannot use
             # the regular add_callback because it may deadlock on
             # _callback_lock.  Blindly insert into self._callbacks.
             # This is safe because the GIL makes list.append atomic.
             # One subtlety is that if the signal interrupted the
             # _callback_lock block in IOLoop.start, we may modify
             # either the old or new version of self._callbacks,
             # but either way will work.
             self._callbacks.append(functools.partial(
                 stack_context.wrap(callback), *args, **kwargs))
Beispiel #19
0
    def test_pre_wrap_with_args(self):
        # Same as test_pre_wrap, but the function takes arguments.
        # Implementation note: The function must not be wrapped in a
        # functools.partial until after it has been passed through
        # stack_context.wrap
        def f1(foo, bar):
            self.assertIn('c1', self.active_contexts)
            self.assertNotIn('c2', self.active_contexts)
            self.stop((foo, bar))

        with StackContext(functools.partial(self.context, 'c1')):
            wrapped = wrap(f1)

        with StackContext(functools.partial(self.context, 'c2')):
            self.add_callback(wrapped, 1, bar=2)

        result = self.wait()
        self.assertEqual(result, (1, 2))
Beispiel #20
0
 def __init__(self, stream, address, request_callback, no_keep_alive=False,
              xheaders=False, protocol=None):
     self.stream = stream
     self.address = address
     # Save the socket's address family now so we know how to
     # interpret self.address even after the stream is closed
     # and its socket attribute replaced with None.
     self.address_family = stream.socket.family
     self.request_callback = request_callback
     self.no_keep_alive = no_keep_alive
     self.xheaders = xheaders
     self.protocol = protocol
     self._clear_request_state()
     # Save stack context here, outside of any request.  This keeps
     # contexts from one request from leaking into the next.
     self._header_callback = stack_context.wrap(self._on_headers)
     self.stream.set_close_callback(self._on_connection_close)
     self.stream.read_until(b"\r\n\r\n", self._header_callback)
Beispiel #21
0
    def set_exit_callback(self, callback):
        """Runs ``callback`` when this process exits.

        The callback takes one argument, the return code of the process.

        This method uses a ``SIGCHILD`` handler, which is a global setting
        and may conflict if you have other libraries trying to handle the
        same signal.  If you are using more than one ``IOLoop`` it may
        be necessary to call `Subprocess.initialize` first to designate
        one ``IOLoop`` to run the signal handlers.

        In many cases a close callback on the stdout or stderr streams
        can be used as an alternative to an exit callback if the
        signal handler is causing a problem.
        """
        self._exit_callback = stack_context.wrap(callback)
        Subprocess.initialize(self.io_loop)
        Subprocess._waiting[self.pid] = self
        Subprocess._try_cleanup_process(self.pid)
Beispiel #22
0
    def wrapper(*args, **kwargs):
        future = TracebackFuture()
        callback, args, kwargs = replacer.replace(
            lambda value=_NO_RESULT: future.set_result(value),
            args, kwargs)

        def handle_error(typ, value, tb):
            future.set_exc_info((typ, value, tb))
            return True
        exc_info = None
        with ExceptionStackContext(handle_error):
            try:
                result = f(*args, **kwargs)
                if result is not None:
                    raise ReturnValueIgnoredError(
                        "@return_future should not be used with functions "
                        "that return values")
            except:
                exc_info = sys.exc_info()
                raise
        if exc_info is not None:
            # If the initial synchronous part of f() raised an exception,
            # go ahead and raise it to the caller directly without waiting
            # for them to inspect the Future.
            raise_exc_info(exc_info)

        # If the caller passed in a callback, schedule it to be called
        # when the future resolves.  It is important that this happens
        # just before we return the future, or else we risk confusing
        # stack contexts with multiple exceptions (one here with the
        # immediate exception, and again when the future resolves and
        # the callback triggers its exception by calling future.result()).
        if callback is not None:
            def run_callback(future):
                result = future.result()
                if result is _NO_RESULT:
                    callback()
                else:
                    callback(future.result())
            future.add_done_callback(wrap(run_callback))
        return future
Beispiel #23
0
    def connect(self, address, callback=None, server_hostname=None):
        """Connects the socket to a remote address without blocking.

        May only be called if the socket passed to the constructor was
        not previously connected.  The address parameter is in the
        same format as for `socket.connect <socket.socket.connect>`,
        i.e. a ``(host, port)`` tuple.  If ``callback`` is specified,
        it will be called when the connection is completed.

        If specified, the ``server_hostname`` parameter will be used
        in SSL connections for certificate validation (if requested in
        the ``ssl_options``) and SNI (if supported; requires
        Python 3.2+).

        Note that it is safe to call `IOStream.write
        <BaseIOStream.write>` while the connection is pending, in
        which case the data will be written as soon as the connection
        is ready.  Calling `IOStream` read methods before the socket is
        connected works on some platforms but is non-portable.
        """
        self._connecting = True
        try:
            self.socket.connect(address)
        except socket.error as e:
            # In non-blocking mode we expect connect() to raise an
            # exception with EINPROGRESS or EWOULDBLOCK.
            #
            # On freebsd, other errors such as ECONNREFUSED may be
            # returned immediately when attempting to connect to
            # localhost, so handle them the same way as an error
            # reported later in _handle_connect.
            if (e.args[0] != errno.EINPROGRESS and
                    e.args[0] not in _ERRNO_WOULDBLOCK):
                gen_log.warning("Connect error on fd %d: %s",
                                self.socket.fileno(), e)
                self.close(exc_info=True)
                return
        self._connect_callback = stack_context.wrap(callback)
        self._add_io_state(self.io_loop.WRITE)
Beispiel #24
0
 def add_timeout(self, deadline, callback):
     timeout = _Timeout(deadline, stack_context.wrap(callback), self)
     heapq.heappush(self._timeouts, timeout)
     return timeout
Beispiel #25
0
 def connect(self, address, callback=None, server_hostname=None):
     # Save the user's callback and run it after the ssl handshake
     # has completed.
     self._ssl_connect_callback = stack_context.wrap(callback)
     self._server_hostname = server_hostname
     super(SSLIOStream, self).connect(address, callback=None)
Beispiel #26
0
 def add_handler(self, fd, handler, events):
     self._handlers[fd] = stack_context.wrap(handler)
     self._impl.register(fd, events | self.ERROR)
Beispiel #27
0
 def set_close_callback(self, callback):
     """Call the given callback when the stream is closed."""
     self._close_callback = stack_context.wrap(callback)
 def library_function(callback):
     # capture the caller's context before introducing our own
     callback = wrap(callback)
     with StackContext(functools.partial(self.context, 'library')):
         self.io_loop.add_callback(
             functools.partial(library_inner_callback, callback))
Beispiel #29
0
 def add_callback(self, callback, *args, **kwargs):
     self.reactor.callFromThread(self._run_callback,
                                 wrap(callback), *args, **kwargs)
Beispiel #30
0
 def _set_read_callback(self, callback):
     assert not self._read_callback, "Already reading"
     self._read_callback = stack_context.wrap(callback)