Exemple #1
0
    def read_until_close(self, callback, streaming_callback=None):
        """Reads all data from the socket until it is closed.

        If a ``streaming_callback`` is given, it will be called with chunks
        of data as they become available, and the argument to the final
        ``callback`` will be empty.  Otherwise, the ``callback`` gets the
        data as an argument.

        Subject to ``max_buffer_size`` limit from `IOStream` constructor if
        a ``streaming_callback`` is not used.
        """
        self._set_read_callback(callback)
        self._streaming_callback = stack_context.wrap(streaming_callback)
        if self.closed():
            if self._streaming_callback is not None:
                self._run_callback(self._streaming_callback,
                                   self._consume(self._read_buffer_size))
            self._run_callback(self._read_callback,
                               self._consume(self._read_buffer_size))
            self._streaming_callback = None
            self._read_callback = None
            return
        self._read_until_close = True
        self._streaming_callback = stack_context.wrap(streaming_callback)
        self._try_inline_read()
Exemple #2
0
 def add_callback(self, callback, *args, **kwargs):
     if thread.get_ident() != self._thread_ident:
         # If we're not on the IOLoop's thread, we need to synchronize
         # with other threads, or waking logic will induce a race.
         with self._callback_lock:
             if self._closing:
                 return
             list_empty = not self._callbacks
             self._callbacks.append(functools.partial(
                 stack_context.wrap(callback), *args, **kwargs))
             if list_empty:
                 # If we're not in the IOLoop's thread, and we added the
                 # first callback to an empty list, we may need to wake it
                 # up (it may wake up on its own, but an occasional extra
                 # wake is harmless).  Waking up a polling IOLoop is
                 # relatively expensive, so we try to avoid it when we can.
                 self._waker.wake()
     else:
         if self._closing:
             return
         # If we're on the IOLoop's thread, we don't need the lock,
         # since we don't need to wake anyone, just add the
         # callback. Blindly insert into self._callbacks. This is
         # safe even from signal handlers because the GIL makes
         # list.append atomic. One subtlety is that if the signal
         # is interrupting another thread holding the
         # _callback_lock block in IOLoop.start, we may modify
         # either the old or new version of self._callbacks, but
         # either way will work.
         self._callbacks.append(functools.partial(
             stack_context.wrap(callback), *args, **kwargs))
Exemple #3
0
 def __init__(self, stream, address, server):
     self.stream = stream
     self.address = address
     self.server = server
     self.headers_callback = stack_context.wrap(self.on_headers)
     self.body_callback = stack_context.wrap(self.on_body)
     self.stream.read_until(',', self.headers_callback)
Exemple #4
0
 def __init__(self, stream, address, request_callback, no_keep_alive=False,
              xheaders=False, headers_callback=None, close_callback=None):
     self.stream = stream
     if self.stream.socket.family not in (socket.AF_INET, socket.AF_INET6):
         # Unix (or other) socket; fake the remote address
         address = ('0.0.0.0', 0)
     self.address = address
     self.request_callback = request_callback
     self.no_keep_alive = no_keep_alive
     self.xheaders = xheaders
     self._request = None
     self._request_finished = False
     # Save stack context here, outside of any request.  This keeps
     # contexts from one request from leaking into the next.
     self._header_callback = stack_context.wrap(self._on_headers)
     if headers_callback:
         self.on_headers = stack_context.wrap(headers_callback)
     else:
         self.on_headers = lambda *args: None
     if close_callback:
         self.on_finish = stack_context.wrap(close_callback)
     else:
         self.on_finish = lambda *args: None
     self.stream.read_until(b("\r\n\r\n"), self._header_callback)
     self._write_callback = None
Exemple #5
0
 def __init__(self, callback, barrier_type, on_exception=None):
   callback = stack_context.wrap(callback)
   on_exception = stack_context.wrap(on_exception)
   # Parent frame is derived class __init__, so get grandparent frame.
   frame = sys._getframe().f_back.f_back
   self._barrier = _Barrier(callback, on_exception, barrier_type, frame)
   self._stack_context = stack_context.ExceptionStackContext(self._barrier.ReportException)
Exemple #6
0
    def read_by_delimiter_until_close(self, callback, streaming_callback=None, delimiter=None):
        """Reads all data from the socket until it is closed by delimiters.

        :param streaming_callback: function called on all chunk
        :param delimite: chunks delimiter
        :param callback:
        """
        self._set_read_callback(callback)
        self._read_delimiter = delimiter
        self._read_delimiter_len = len(delimiter)
        self._streaming_callback = stack_context.wrap(streaming_callback)

        if self.closed():

            if self._streaming_callback is not None:

                self._run_callback(self._streaming_callback,
                                   self._consume(self._read_buffer_size))


            self._run_callback(self._read_callback,
                               self._consume(self._read_buffer_size))
            self._streaming_callback = None
            self._read_callback = None
            return

        self._read_until_close = True
        self._streaming_callback = stack_context.wrap(streaming_callback)
        self._try_inline_read()
    def __init__(self,stream,clientid,address,receiveDataCallback,closeCallback,**kwargs):
        '''
        @param clientid,int16
        '''
        self.clientid = clientid

        self.stream = stream
        self.address = address
        self.MASK1 = kwargs.get('mask1',0x59) 
        self.MASK2 = kwargs.get('mask2',0x7a)

        self._closeCallback = closeCallback
        self._receive_callback = stack_context.wrap(receiveDataCallback)

        self._masklen = 2 
        self._read_buffer = []

        stream.set_close_callback(self.loseConnection)

        # Save stack context here, outside of any request.  This keeps
        # contexts from one request from leaking into the next.
        self._read_callback = stack_context.wrap(self._on_read_start_mask)
        self._read_length_callback = stack_context.wrap(self._on_read_length)
        self._read_body_callback = stack_context.wrap(self._on_read_body)

        self.stream.read_bytes(self._masklen,self._read_callback)
Exemple #8
0
def get_lj_post_task_list(task_cfg, task_begin_handle=None, task_end_handle=None):
    task_begin_handle = stack_context.wrap(task_begin_handle)
    task_end_handle = stack_context.wrap(task_end_handle)

    raw_accs_iter = get_items.get_random_infinite_items(task_cfg.accs, is_csv=True)
    if task_cfg.tags is not None:
        tags_iter = get_items.get_random_infinite_items(task_cfg.tags)
    else:
        tags_iter = None
    title_and_content_iter = get_items.get_title_and_content(
        get_items.get_random_infinite_items, task_cfg.titles, task_cfg.content
    )

    def next_acc():
        if "lj:0" == task_cfg.acc_fmt:
            while True:
                acc_row = next(raw_accs_iter)

                if len(acc_row) != 4:
                    raise NotImplementedError("invalid or not implemented account format")

                email, email_password, username, password = acc_row

                return username, password, acc_row

        # if 'lj:...' == task_cfg.acc_fmt:
        #  ...
        #  return

        raise NotImplementedError("not implemented account format")

    for task_i in range(task_cfg.count):
        task = Task()

        task.i = task_i
        task.username, task.password, task._acc_row = next_acc()
        task.blog_id = "lj:{}".format(task.username)
        task.title, task.content = next(title_and_content_iter)
        task.ua_name = task_cfg.ua_name
        task.proxy_kwargs = task_cfg.proxy_kwargs

        if tags_iter is not None:
            tags_list = []
            for tag_i in range(max(round(random.gauss(TAGS_RANDOM_MU, TAGS_RANDOM_SIGMA)), 0)):
                tag = next(tags_iter)
                if tag in tags_list:
                    continue
                tags_list.append(tag)
            task.tags = ", ".join(tags_list)
        else:
            task.tags = None

        task.acc_save = lambda _task=task: lj_acc_save(task_cfg, _task)

        task.task_begin_handle = task_begin_handle
        task.task_end_handle = task_end_handle

        yield task
Exemple #9
0
 def add_callback(self, callback, *args, **kwargs):
     if self.closing:
         raise RuntimeError("IOLoop is closing")
     if kwargs:
         self.asyncio_loop.call_soon_threadsafe(
             functools.partial(self._run_callback, stack_context.wrap(callback), *args, **kwargs)
         )
     else:
         self.asyncio_loop.call_soon_threadsafe(self._run_callback, stack_context.wrap(callback), *args)
Exemple #10
0
    def get(self, callback, timeout=None):
        '''
        Wait for the RPC associated with this :class:`RPCResponseFuture`
        to return a result.  When the result is received, resolve the
        task by calling the passed in ``callback``.

        :param callback: The callback that will be called with the RPC
            response upon completion of the RPC.  It is recommended that
            this not be passed in directly, but rather that
            :meth:`~.get` be called as a function passed to
            :class:`tornado.gen.Task`.

        :param timeout: The amount of time to wait before raising an
            :exc:`RPCTimeoutError` to indicate that the RPC has timed
            out.  This can be a number or a :class:`timedelta`
            object.  If it is a number, it will be treated as
            seconds.
        '''

        self.get_time = datetime.now()

        if self.response_received:
            logger.info('Response has already been received, return '
                        'the value immediately.')
            callback(self.response)
        else:
            callback = stack_context.wrap(callback)
            if self.timeout and not timeout:
                timeout = self.timeout
            elif not self.timeout and not timeout:
                timeout = timedelta(seconds=6)

            key = uuid.uuid4()
            self.wait_callback = yield gen.Callback(key)

            logger.info('Response has not been received yet.  Adding '
                        'timeout to the io_loop in case the response '
                        'times out.')

            if isinstance(timeout, numbers.Real):
                timeout = timedelta(seconds=timeout)

            timeout_callback = stack_context.wrap(self.timeout_callback)
            self.io_loop.add_timeout(timeout, timeout_callback)

            logger.info('Waiting for the response.')
            yield gen.Wait(key)

            if self.timed_out:
                raise RPCTimeoutError('Future waiting for message with cid: '
                                      '"%s" timed out' % str(self.cid))
            elif self.response_received:
                logger.info('Response received successfully.')
                callback(self.response)
            else:
                raise Exception("Neither timed out nor response received")
 def add_job(self, func, cb, exception_cb, prio=10):
     try:
         ThreadPoolExecutor.count += 1
         self.events.put((
             (prio, ThreadPoolExecutor.count),
             (func, stack_context.wrap(cb), stack_context.wrap(exception_cb))
         ))
     except Exception as e:
         jobs_log.exception('cannot put job to queue')
         IOLoop.instance().add_callback(partial(exception_cb, e))
Exemple #12
0
 def __init__(self, connections, object_callback=None,
              finished_callback=None):
     self._object_callback = stack_context.wrap(object_callback)
     self._finished_callback = stack_context.wrap(finished_callback)
     # Connections that have not finished searching
     self._connections = set(connections)
     # Connections without an outstanding blast request
     self._blocking = set(connections)
     self._started = False
     self._paused = False
Exemple #13
0
    def __init__(self, client, request, release_callback,
                 final_callback, max_buffer_size, tcp_client,
                 max_header_size, max_body_size):
        self.io_loop = IOLoop.current()
        self.start_time = self.io_loop.time()
        self.client = client
        self.request = request
        self.release_callback = release_callback
        self.final_callback = final_callback
        self.max_buffer_size = max_buffer_size
        self.tcp_client = tcp_client
        self.max_header_size = max_header_size
        self.max_body_size = max_body_size
        self.code = None
        self.headers = None
        self.chunks = []
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        self._sockaddr = None
        with stack_context.ExceptionStackContext(self._handle_exception):
            self.parsed = urlparse.urlsplit(_unicode(self.request.url))
            if self.parsed.scheme not in ("http", "https"):
                raise ValueError("Unsupported url scheme: %s" %
                                 self.request.url)
            # urlsplit results have hostname and port results, but they
            # didn't support ipv6 literals until python 2.7.
            netloc = self.parsed.netloc
            if "@" in netloc:
                userpass, _, netloc = netloc.rpartition("@")
            host, port = httputil.split_host_and_port(netloc)
            if port is None:
                port = 443 if self.parsed.scheme == "https" else 80
            if re.match(r'^\[.*\]$', host):
                # raw ipv6 addresses in urls are enclosed in brackets
                host = host[1:-1]
            self.parsed_hostname = host  # save final host for _on_connect

            if request.allow_ipv6 is False:
                af = socket.AF_INET
            else:
                af = socket.AF_UNSPEC

            ssl_options = self._get_ssl_options(self.parsed.scheme)

            timeout = min(self.request.connect_timeout, self.request.request_timeout)
            if timeout:
                self._timeout = self.io_loop.add_timeout(
                    self.start_time + timeout,
                    stack_context.wrap(functools.partial(self._on_timeout, "while connecting")))
            fut = self.tcp_client.connect(host, port, af=af,
                                          ssl_options=ssl_options,
                                          max_buffer_size=self.max_buffer_size)
            fut.add_done_callback(stack_context.wrap(self._on_connect))
Exemple #14
0
 def write(self, chunk, callback=None):
     # ZMQWEB NOTE: This method is overriden from the base class.
     msg_list = self._build_reply()
     msg_list.extend([b'DATA', chunk])
     logging.debug('Sending write: %r', msg_list)
     self.stream.send_multipart(msg_list)
     # ZMQWEB NOTE: We don't want to permanently register an on_send callback
     # with the stream, so we just call the callback immediately.
     if callback is not None:
         try:
             stack_context.wrap(callback)()
         except:
             logging.error('Unexpected exception in write callback', exc_info=True)
Exemple #15
0
    def authorize_redirect(self, callback_uri=None, extra_params=None,
                           http_client=None, callback=None):
        """Redirects the user to obtain OAuth authorization for this service.

        The ``callback_uri`` may be omitted if you have previously
        registered a callback URI with the third-party service. For
        some services, you must use a previously-registered callback
        URI and cannot specify a callback via this method.

        This method sets a cookie called ``_oauth_request_token`` which is
        subsequently used (and cleared) in `get_authenticated_user` for
        security purposes.

        This method is asynchronous and must be called with ``await``
        or ``yield`` (This is different from other ``auth*_redirect``
        methods defined in this module). It calls
        `.RequestHandler.finish` for you so you should not write any
        other response after it returns.

        .. versionchanged:: 3.1
           Now returns a `.Future` and takes an optional callback, for
           compatibility with `.gen.coroutine`.

        .. deprecated:: 5.1

           The ``callback`` argument is deprecated and will be removed in 6.0.
           Use the returned awaitable object instead.

        """
        if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False):
            raise Exception("This service does not support oauth_callback")
        if http_client is None:
            http_client = self.get_auth_http_client()
        if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a":
            fut = http_client.fetch(
                self._oauth_request_token_url(callback_uri=callback_uri,
                                              extra_params=extra_params))
            fut.add_done_callback(wrap(functools.partial(
                self._on_request_token,
                self._OAUTH_AUTHORIZE_URL,
                callback_uri,
                callback)))
        else:
            fut = http_client.fetch(self._oauth_request_token_url())
            fut.add_done_callback(
                wrap(functools.partial(
                    self._on_request_token, self._OAUTH_AUTHORIZE_URL,
                    callback_uri,
                    callback)))
Exemple #16
0
 def __init__(self, stream, address, request_callback, no_keep_alive=False,
              xheaders=False, close_callback=None, headers_callback=None):
     self.stream = stream
     self.address = address
     self.request_callback = request_callback
     self.no_keep_alive = no_keep_alive
     self.xheaders = xheaders
     self._request = None
     self._request_finished = False
     # Save stack context here, outside of any request.  This keeps
     # contexts from one request from leaking into the next.
     self._header_callback = stack_context.wrap(self._on_headers)
     self._headers_callback = stack_context.wrap(headers_callback)
     self._close_callback = stack_context.wrap(close_callback)
     self.stream.read_until("\r\n\r\n", self._header_callback)
Exemple #17
0
 def read_until_regex(self, regex, callback, failure_callback=None):
     """Call callback when we read the given regex pattern."""
     assert not self._read_callback, "Already reading"
     self._read_regex = re.compile(regex)
     self._read_callback = stack_context.wrap(callback)
     if failure_callback:
         self._read_failure_callback = stack_context.wrap(failure_callback)
     while True:
         # See if we've already got the data from a previous read
         if self._read_from_buffer():
             return
         self._check_closed()
         if self._read_to_buffer() == 0:
             break
     self._add_io_state(self.io_loop.READ)
Exemple #18
0
 def read_bytes(self, num_bytes, callback, failure_callback=None):
     """Call callback when we read the given number of bytes."""
     assert not self._blocking
     assert not self._read_callback, "Already reading"
     assert isinstance(num_bytes, int)
     self._read_bytes = num_bytes
     self._read_callback = stack_context.wrap(callback)
     self._read_failure_callback = stack_context.wrap(failure_callback)
     while True:
         if self._read_from_buffer():
             return
         self._check_closed()
         if self._read_to_buffer() == 0:
             break
     self._add_io_state(self.io_loop.READ)
 def read_until_close(self, callback, streaming_callback=None):
     """ 读取直到关闭。
     如果streaming_callback不为空,则它将处理所有的数据,callback得到的参数将为空。 """
     self._set_read_callback(callback)
     self._streaming_callback = stack_context.wrap(streaming_callback)
     if self.closed(): # 如果已经关闭则一次性消费完整个_read_buffer然后返回
         if self._streaming_callback is not None:
             self._run_callback(self._streaming_callback, self._consume(self._read_buffer_size))
         self._run_callback(self._read_callback, self._consume(self._read_buffer_size))
         self._streaming_callback = None
         self._read_callback = None
         return
     self._read_until_close = True
     self._streaming_callback = stack_context.wrap(streaming_callback) # 设置好_streaming_callback后注册io_loop
     self._add_io_state(self.io_loop.READ)
Exemple #20
0
    def fetch(self, request, callback, **kwargs):
        if not isinstance(request, HTTPRequest):
            request = HTTPRequest(url=request, **kwargs)
        # We're going to modify this (to add Host, Accept-Encoding, etc),
        # so make sure we don't modify the caller's object.  This is also
        # where normal dicts get converted to HTTPHeaders objects.
        request.headers = httputil.HTTPHeaders(request.headers)
        callback = stack_context.wrap(callback)

        key = object()
        self.queue.append((key, request, callback))

        if not len(self.active) < self.max_clients:
            timeout_handle = self.io_loop.add_timeout(
                time.time() + min(request.connect_timeout,
                                  request.request_timeout),
                functools.partial(self._on_timeout, key))
        else:
            timeout_handle = None

        self.waiting[key] = (request, callback, timeout_handle)
        self._process_queue()
        if self.queue:
            logging.debug(
                'max_clients limit reached, request queued. '
                '%d active, %d queued requests.' % (
                    len(self.active), len(self.queue))
            )
Exemple #21
0
 def call_at(self, when, callback, *args, **kwargs):
     # asyncio.call_at supports *args but not **kwargs, so bind them here.
     # We do not synchronize self.time and asyncio_loop.time, so
     # convert from absolute to relative.
     return self.asyncio_loop.call_later(
         max(0, when - self.time()), self._run_callback,
         functools.partial(stack_context.wrap(callback), *args, **kwargs))
Exemple #22
0
 def __init__(self, address, close_callback):
     self._close_callback = stack_context.wrap(close_callback)
     self._finished = False  # No more results
     self._closed = False    # Connection closed
     self.address = address
     self.control = ControlConnection(self.close)
     self.blast = BlastConnection(self.close)
Exemple #23
0
 def call_at(self, deadline, callback, *args, **kwargs):
     timeout = _Timeout(
         deadline,
         functools.partial(stack_context.wrap(callback), *args, **kwargs),
         self)
     heapq.heappush(self._timeouts, timeout)
     return timeout
Exemple #24
0
    def write(self, data, callback=None):
        """Write the given data to this stream.

        If callback is given, we call it when all of the buffered write
        data has been successfully written to the stream. If there was
        previously buffered write data and an old write callback, that
        callback is simply overwritten with this new callback.
        """
        assert isinstance(data, bytes_type)
        self._check_closed()
        # We use bool(_write_buffer) as a proxy for write_buffer_size>0,
        # so never put empty strings in the buffer.
        if data:
            # Break up large contiguous strings before inserting them in the
            # write buffer, so we don't have to recopy the entire thing
            # as we slice off pieces to send to the socket.
            WRITE_BUFFER_CHUNK_SIZE = 128 * 1024
            if len(data) > WRITE_BUFFER_CHUNK_SIZE:
                for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
                    self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
            else:
                self._write_buffer.append(data)
        self._write_callback = stack_context.wrap(callback)
        if not self._connecting:
            self._handle_write()
            if self._write_buffer:
                self._add_io_state(self.io_loop.WRITE)
            self._maybe_add_error_listener()
Exemple #25
0
    def send_message(self, args, callback=None):

        command = args[0]

        if 'SUBSCRIBE' in command:
            raise NotImplementedError('Not yet.')

        # Do not allow the commands, affecting the execution of other commands,
        # to be used on shared connection.
        if command in ('WATCH', 'MULTI'):
            if self.is_shared():
                raise Exception('Command %s is not allowed while connection '
                                'is shared!' % command)
            if command == 'WATCH':
                self._watch.add(args[1])
            if command == 'MULTI':
                self._multi = True

        # monitor transaction state, to unlock correctly
        if command in ('EXEC', 'DISCARD', 'UNWATCH'):
            if command in ('EXEC', 'DISCARD'):
                self._multi = False
            self._watch.clear()

        self.stream.write(self.format_message(args))

        future = Future()

        if callback is not None:
            future.add_done_callback(stack_context.wrap(callback))

        self.callbacks.append(future.set_result)

        return future
Exemple #26
0
    def connect(self, address, callback=None):
        """Connects the socket to a remote address without blocking.

        May only be called if the socket passed to the constructor was
        not previously connected.  The address parameter is in the
        same format as for socket.connect, i.e. a (host, port) tuple.
        If callback is specified, it will be called when the
        connection is completed.

        Note that it is safe to call IOStream.write while the
        connection is pending, in which case the data will be written
        as soon as the connection is ready.  Calling IOStream read
        methods before the socket is connected works on some platforms
        but is non-portable.
        """
        self._connecting = True
        try:
            self.socket.connect(address)
        except socket.error as e:
            # In non-blocking mode we expect connect() to raise an
            # exception with EINPROGRESS or EWOULDBLOCK.
            #
            # On freebsd, other errors such as ECONNREFUSED may be
            # returned immediately when attempting to connect to
            # localhost, so handle them the same way as an error
            # reported later in _handle_connect.
            if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
                logging.warning("Connect error on fd %d: %s",
                                self.socket.fileno(), e)
                self.close()
                return
        self._connect_callback = stack_context.wrap(callback)
        self._add_io_state(self.io_loop.WRITE)
Exemple #27
0
 def on_recv(self, callback, copy=True):
     """Register a callback for when a message is ready to recv.
     
     There can be only one callback registered at a time, so each
     call to `on_recv` replaces previously registered callbacks.
     
     on_recv(None) disables recv event polling.
     
     Use on_recv_stream(callback) instead, to register a callback that will receive
     both this ZMQStream and the message, instead of just the message.
     
     Parameters
     ----------
     
     callback : callable
         callback must take exactly one argument, which will be a
         list, as returned by socket.recv_multipart()
         if callback is None, recv callbacks are disabled.
     copy : bool
         copy is passed directly to recv, so if copy is False,
         callback will receive Message objects. If copy is True,
         then callback will receive bytes/str objects.
     
     Returns : None
     """
     
     self._check_closed()
     assert callback is None or callable(callback)
     self._recv_callback = stack_context.wrap(callback)
     self._recv_copy = copy
     if callback is None:
         self._drop_io_state(self.io_loop.READ)
     else:
         self._add_io_state(self.io_loop.READ)
Exemple #28
0
 def fetch(self, request, callback, **kwargs):
     if not isinstance(request, HTTPRequest):
         request = HTTPRequest(url=request, **kwargs)
     request = _RequestProxy(request, self.defaults)
     self._requests.append((request, stack_context.wrap(callback)))
     self._process_queue()
     self._set_timeout(0)
Exemple #29
0
    def send_message(self, message, with_last_error=False, callback=None):
        """Say something to Mongo.

        Raises ConnectionFailure if the message cannot be sent. Raises
        OperationFailure if `with_last_error` is ``True`` and the
        response to the getLastError call returns an error. Return the
        response from lastError, or ``None`` if `with_last_error`
        is ``False``.

        :Parameters:
          - `message`: message to send
          - `with_last_error`: check getLastError status after sending the
            message
        """
        if self._callback is not None:
            raise ProgrammingError('connection already in use')

        if self.closed():
            if self._autoreconnect:
                self._connect()
            else:
                raise InterfaceError('connection is closed and autoreconnect is false')

        self._callback = stack_context.wrap(callback)
        self._check_response = with_last_error

        with stack_context.StackContext(self.close_on_error):
            self.__send_message(message, with_last_error=with_last_error)
Exemple #30
0
    def set_close_callback(self, callback):
        """Sets a callback that will be run when the connection is closed.

        .. deprecated:: 4.0
            Use `.HTTPMessageDelegate.on_connection_close` instead.
        """
        self._close_callback = stack_context.wrap(callback)
Exemple #31
0
 def streaming_callback(self, value):
     self._streaming_callback = stack_context.wrap(value)
Exemple #32
0
 def body_producer(self, value):
     self._body_producer = stack_context.wrap(value)
Exemple #33
0
    def fetch(self, request, callback=None, raise_error=True, **kwargs):
        """Executes a request, asynchronously returning an `HTTPResponse`.

        The request may be either a string URL or an `HTTPRequest` object.
        If it is a string, we construct an `HTTPRequest` using any additional
        kwargs: ``HTTPRequest(request, **kwargs)``

        This method returns a `.Future` whose result is an
        `HTTPResponse`. By default, the ``Future`` will raise an
        `HTTPError` if the request returned a non-200 response code
        (other errors may also be raised if the server could not be
        contacted). Instead, if ``raise_error`` is set to False, the
        response will always be returned regardless of the response
        code.

        If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
        In the callback interface, `HTTPError` is not automatically raised.
        Instead, you must check the response's ``error`` attribute or
        call its `~HTTPResponse.rethrow` method.
        """
        if self._closed:
            raise RuntimeError("fetch() called on closed AsyncHTTPClient")
        if not isinstance(request, HTTPRequest):
            request = HTTPRequest(url=request, **kwargs)
        else:
            if kwargs:
                raise ValueError(
                    "kwargs can't be used if request is an HTTPRequest object")
        # We may modify this (to add Host, Accept-Encoding, etc),
        # so make sure we don't modify the caller's object.  This is also
        # where normal dicts get converted to HTTPHeaders objects.
        request.headers = httputil.HTTPHeaders(request.headers)
        request = _RequestProxy(request, self.defaults)
        future = TracebackFuture()
        if callback is not None:
            callback = stack_context.wrap(callback)

            def handle_future(future):
                exc = future.exception()
                if isinstance(exc, HTTPError) and exc.response is not None:
                    response = exc.response
                elif exc is not None:
                    response = HTTPResponse(request,
                                            599,
                                            error=exc,
                                            request_time=time.time() -
                                            request.start_time)
                else:
                    response = future.result()
                self.io_loop.add_callback(callback, response)

            future.add_done_callback(handle_future)

        def handle_response(response):
            if raise_error and response.error:
                future.set_exception(response.error)
            else:
                future.set_result(response)

        self.fetch_impl(request, handle_response)
        return future
Exemple #34
0
 def set_close_callback(self, callback):
     """Call the given callback when the stream is closed."""
     self._close_callback = stack_context.wrap(callback)
Exemple #35
0
 def connect(self, address, callback=None, server_hostname=None):
     # Save the user's callback and run it after the ssl handshake
     # has completed.
     self._ssl_connect_callback = stack_context.wrap(callback)
     self._server_hostname = server_hostname
     super(SSLIOStream, self).connect(address, callback=None)
Exemple #36
0
 def add_parse_callback(self, callback):
     """Adds a parse callback, to be invoked when option parsing is done."""
     self._parse_callbacks.append(stack_context.wrap(callback))
Exemple #37
0
 def add_callback(self, callback):
     """Calls the given callback on the next I/O loop iteration."""
     self._callbacks.add(stack_context.wrap(callback))
     self._wake()
Exemple #38
0
 def add_timeout(self, deadline, callback):
     """Calls the given callback at the time deadline from the I/O loop."""
     timeout = _Timeout(deadline, stack_context.wrap(callback))
     bisect.insort(self._timeouts, timeout)
     return timeout
Exemple #39
0
 def add_timeout(self, deadline, callback):
     timeout = _Timeout(deadline, stack_context.wrap(callback), self)
     self._timeouts.add(timeout)
     return timeout
Exemple #40
0
 def add_handler(self, fd, handler, events):
     self._handlers[fd] = stack_context.wrap(handler)
     self._impl.register(fd, events | self.ERROR)
Exemple #41
0
 def write(self, chunk, callback=None):
     """Writes a chunk of output to the stream."""
     assert self._request, "Request closed"
     if not self.stream.closed():
         self._write_callback = stack_context.wrap(callback)
         self.stream.write(chunk, self._on_write_complete)
 def write_headers(self, start_line, headers, chunk=None, callback=None):
     """Implements `.HTTPConnection.write_headers`."""
     lines = []
     if self.is_client:
         self._request_start_line = start_line
         lines.append(
             utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1])))
         # Client requests with a non-empty body must have either a
         # Content-Length or a Transfer-Encoding.
         self._chunking_output = (start_line.method
                                  in ('POST', 'PUT', 'PATCH')
                                  and 'Content-Length' not in headers
                                  and 'Transfer-Encoding' not in headers)
     else:
         self._response_start_line = start_line
         lines.append(
             utf8('HTTP/1.1 %d %s' % (start_line[1], start_line[2])))
         self._chunking_output = (
             # TODO: should this use
             # self._request_start_line.version or
             # start_line.version?
             self._request_start_line.version == 'HTTP/1.1' and
             # 304 responses have no body (not even a zero-length body), and so
             # should not have either Content-Length or Transfer-Encoding.
             # headers.
             start_line.code != 304 and
             # No need to chunk the output if a Content-Length is specified.
             'Content-Length' not in headers and
             # Applications are discouraged from touching Transfer-Encoding,
             # but if they do, leave it alone.
             'Transfer-Encoding' not in headers)
         # If a 1.0 client asked for keep-alive, add the header.
         if (self._request_start_line.version == 'HTTP/1.0'
                 and (self._request_headers.get('Connection', '').lower()
                      == 'keep-alive')):
             headers['Connection'] = 'Keep-Alive'
     if self._chunking_output:
         headers['Transfer-Encoding'] = 'chunked'
     if (not self.is_client and (self._request_start_line.method == 'HEAD'
                                 or start_line.code == 304)):
         self._expected_content_remaining = 0
     elif 'Content-Length' in headers:
         self._expected_content_remaining = int(headers['Content-Length'])
     else:
         self._expected_content_remaining = None
     lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()])
     for line in lines:
         if b'\n' in line:
             raise ValueError('Newline in header: ' + repr(line))
     future = None
     if self.stream.closed():
         future = self._write_future = Future()
         future.set_exception(iostream.StreamClosedError())
         future.exception()
     else:
         if callback is not None:
             self._write_callback = stack_context.wrap(callback)
         else:
             future = self._write_future = Future()
         data = b"\r\n".join(lines) + b"\r\n\r\n"
         if chunk:
             data += self._format_chunk(chunk)
         self._pending_write = self.stream.write(data)
         self._pending_write.add_done_callback(self._on_write_complete)
     return future
 def f1():
     with NullContext():
         wrapped = wrap(f2)
     with StackContext(functools.partial(self.context, 'c2')):
         wrapped()
Exemple #44
0
 def header_callback(self, value):
     self._header_callback = stack_context.wrap(value)
Exemple #45
0
 def prepare_curl_callback(self, value):
     self._prepare_curl_callback = stack_context.wrap(value)
Exemple #46
0
 def fetch(self, request, callback, **kwargs):
     if not isinstance(request, HTTPRequest):
         request = HTTPRequest(url=request, **kwargs)
     self._requests.append((request, stack_context.wrap(callback)))
     self._process_queue()
     self._set_timeout(0)
Exemple #47
0
    def _on_connect(self):
        self._remove_timeout()
        if self.final_callback is None:
            return
        if self.request.request_timeout:
            self._timeout = self.io_loop.add_timeout(
                self.start_time + self.request.request_timeout,
                stack_context.wrap(self._on_timeout))
        if (self.request.method not in self._SUPPORTED_METHODS
                and not self.request.allow_nonstandard_methods):
            raise KeyError("unknown method %s" % self.request.method)
        for key in (
                'network_interface',
                #'proxy_host', 'proxy_port',
                'proxy_username',
                'proxy_password'):
            if getattr(self.request, key, None):
                raise NotImplementedError('%s not supported' % key)
        # KA: here is the only changes
        #if "Connection" not in self.request.headers:

        self.request.headers["Connection"] = "keep-alive"
        if "Host" not in self.request.headers:
            if '@' in self.parsed.netloc:
                self.request.headers["Host"] = self.parsed.netloc.rpartition(
                    '@')[-1]
            else:
                self.request.headers["Host"] = self.parsed.netloc
        username, password = None, None
        if self.parsed.username is not None:
            username, password = self.parsed.username, self.parsed.password
        elif self.request.auth_username is not None:
            username = self.request.auth_username
            password = self.request.auth_password or ''
        if username is not None:
            if self.request.auth_mode not in (None, "basic"):
                raise ValueError("unsupported auth_mode %s",
                                 self.request.auth_mode)
            auth = utf8(username) + b":" + utf8(password)
            self.request.headers["Authorization"] = (b"Basic " +
                                                     base64.b64encode(auth))
        if self.request.user_agent:
            self.request.headers["User-Agent"] = self.request.user_agent
        if not self.request.allow_nonstandard_methods:
            if self.request.method in ("POST", "PATCH", "PUT"):
                if self.request.body is None:
                    raise AssertionError(
                        'Body must not be empty for "%s" request' %
                        self.request.method)
            else:
                if self.request.body is not None:
                    raise AssertionError(
                        'Body must be empty for "%s" request' %
                        self.request.method)
        if self.request.body is not None:
            self.request.headers["Content-Length"] = str(len(
                self.request.body))
        if (self.request.method == "POST"
                and "Content-Type" not in self.request.headers):
            self.request.headers[
                "Content-Type"] = "application/x-www-form-urlencoded"
        if self.request.decompress_response:
            self.request.headers["Accept-Encoding"] = "gzip"

        if self.proxy_host:
            req_path = self.request.url
        else:
            req_path = ((self.parsed.path or '/') + (
                ('?' + self.parsed.query) if self.parsed.query else ''))

        request_lines = [
            utf8("%s %s HTTP/1.1" % (self.request.method, req_path))
        ]
        for k, v in self.request.headers.get_all():
            line = utf8(k) + b": " + utf8(v)
            if b'\n' in line:
                raise ValueError('Newline in header: ' + repr(line))
            request_lines.append(line)
        request_str = b"\r\n".join(request_lines) + b"\r\n\r\n"
        if self.request.body is not None:
            request_str += self.request.body
        self.stream.set_nodelay(True)
        self.stream.write(request_str)
        self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)
Exemple #48
0
 def add_callback(self, callback, *args, **kwargs):
     if self.closing:
         raise RuntimeError("IOLoop is closing")
     self.asyncio_loop.call_soon_threadsafe(
         self._run_callback,
         functools.partial(stack_context.wrap(callback), *args, **kwargs))
Exemple #49
0
 def result_callback(self, key):
     return stack_context.wrap(_argument_adapter(
         functools.partial(self.set_result, key)))
Exemple #50
0
 def _set_read_callback(self, callback):
     assert not self._read_callback, "Already reading"
     self._read_callback = stack_context.wrap(callback)
Exemple #51
0
        try:
            self.socket.connect(address)
        except socket.error, e:
            # In non-blocking mode we expect connect() to raise an
            # exception with EINPROGRESS or EWOULDBLOCK.
            #
            # On freebsd, other errors such as ECONNREFUSED may be
            # returned immediately when attempting to connect to
            # localhost, so handle them the same way as an error
            # reported later in _handle_connect.
            if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
                gen_log.warning("Connect error on fd %d: %s",
                                self.socket.fileno(), e)
                self.close()
                return
        self._connect_callback = stack_context.wrap(callback)
        self._add_io_state(self.io_loop.WRITE)

    def _handle_connect(self):
        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
        if err != 0:
            self.error = socket.error(err, os.strerror(err))
            # IOLoop implementations may vary: some of them return
            # an error state before the socket becomes writable, so
            # in that case a connection failure would be handled by the
            # error path in _handle_events instead of here.
            gen_log.warning("Connect error on fd %d: %s",
                            self.socket.fileno(), errno.errorcode[err])
            self.close()
            return
        if self._connect_callback is not None:
 def write(self, chunk, callback=None):
     """Writes a chunk of output to the stream."""
     if not self.stream.closed():
         self._write_callback = stack_context.wrap(callback)
         # 调用 IOStream的 write()方法
         self.stream.write(chunk, self._on_write_complete)
 def _on_connect(self, stream):
     if self.final_callback is None:
         # final_callback is cleared if we've hit our timeout.
         stream.close()
         return
     self.stream = stream
     self.stream.set_close_callback(self.on_connection_close)
     self._remove_timeout()
     if self.final_callback is None:
         return
     if self.request.request_timeout:
         self._timeout = self.io_loop.add_timeout(
             self.start_time + self.request.request_timeout,
             stack_context.wrap(self._on_timeout))
     if (self.request.method not in self._SUPPORTED_METHODS and
             not self.request.allow_nonstandard_methods):
         raise KeyError("unknown method %s" % self.request.method)
     for key in ('network_interface',
                 'proxy_host', 'proxy_port',
                 'proxy_username', 'proxy_password'):
         if getattr(self.request, key, None):
             raise NotImplementedError('%s not supported' % key)
     if "Connection" not in self.request.headers:
         self.request.headers["Connection"] = "close"
     if "Host" not in self.request.headers:
         if '@' in self.parsed.netloc:
             self.request.headers["Host"] = self.parsed.netloc.rpartition('@')[-1]
         else:
             self.request.headers["Host"] = self.parsed.netloc
     username, password = None, None
     if self.parsed.username is not None:
         username, password = self.parsed.username, self.parsed.password
     elif self.request.auth_username is not None:
         username = self.request.auth_username
         password = self.request.auth_password or ''
     if username is not None:
         if self.request.auth_mode not in (None, "basic"):
             raise ValueError("unsupported auth_mode %s",
                              self.request.auth_mode)
         auth = utf8(username) + b":" + utf8(password)
         self.request.headers["Authorization"] = (b"Basic " +
                                                  base64.b64encode(auth))
     if self.request.user_agent:
         self.request.headers["User-Agent"] = self.request.user_agent
     if not self.request.allow_nonstandard_methods:
         # Some HTTP methods nearly always have bodies while others
         # almost never do. Fail in this case unless the user has
         # opted out of sanity checks with allow_nonstandard_methods.
         body_expected = self.request.method in ("POST", "PATCH", "PUT")
         body_present = (self.request.body is not None or
                         self.request.body_producer is not None)
         if ((body_expected and not body_present) or
                 (body_present and not body_expected)):
             raise ValueError(
                 'Body must %sbe None for method %s (unless '
                 'allow_nonstandard_methods is true)' %
                 ('not ' if body_expected else '', self.request.method))
     if self.request.expect_100_continue:
         self.request.headers["Expect"] = "100-continue"
     if self.request.body is not None:
         # When body_producer is used the caller is responsible for
         # setting Content-Length (or else chunked encoding will be used).
         self.request.headers["Content-Length"] = str(len(
             self.request.body))
     if (self.request.method == "POST" and
             "Content-Type" not in self.request.headers):
         self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
     if self.request.decompress_response:
         self.request.headers["Accept-Encoding"] = "gzip"
     req_path = ((self.parsed.path or '/') +
                 (('?' + self.parsed.query) if self.parsed.query else ''))
     self.connection = self._create_connection(stream)
     start_line = httputil.RequestStartLine(self.request.method,
                                            req_path, '')
     self.connection.write_headers(start_line, self.request.headers)
     if self.request.expect_100_continue:
         self._read_response()
     else:
         self._write_body(True)
class IOStream(object):
    """A utility class to write to and read from a non-blocking socket.

    We support three methods: write(), read_until(), and read_bytes().
    All of the methods take callbacks (since writing and reading are
    non-blocking and asynchronous). read_until() reads the socket until
    a given delimiter, and read_bytes() reads until a specified number
    of bytes have been read from the socket.

    The socket parameter may either be connected or unconnected.  For
    server operations the socket is the result of calling socket.accept().
    For client operations the socket is created with socket.socket(),
    and may either be connected before passing it to the IOStream or
    connected with IOStream.connect.

    A very simple (and broken) HTTP client using this class:

        from tornado import ioloop
        from tornado import iostream
        import socket

        def send_request():
            stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n")
            stream.read_until("\r\n\r\n", on_headers)

        def on_headers(data):
            headers = {}
            for line in data.split("\r\n"):
               parts = line.split(":")
               if len(parts) == 2:
                   headers[parts[0].strip()] = parts[1].strip()
            stream.read_bytes(int(headers["Content-Length"]), on_body)

        def on_body(data):
            print data
            stream.close()
            ioloop.IOLoop.instance().stop()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        stream = iostream.IOStream(s)
        stream.connect(("friendfeed.com", 80), send_request)
        ioloop.IOLoop.instance().start()

    """
    def __init__(self,
                 socket,
                 io_loop=None,
                 max_buffer_size=104857600,
                 read_chunk_size=4096):
        self.socket = socket
        self.socket.setblocking(False)
        self.io_loop = io_loop or ioloop.IOLoop.instance()
        self.max_buffer_size = max_buffer_size
        self.read_chunk_size = read_chunk_size
        self._read_buffer = collections.deque()
        self._write_buffer = collections.deque()
        self._write_buffer_frozen = False
        self._read_delimiter = None
        self._read_bytes = None
        self._read_callback = None
        self._write_callback = None
        self._close_callback = None
        self._connect_callback = None
        self._connecting = False
        self._state = self.io_loop.ERROR
        with stack_context.NullContext():
            self.io_loop.add_handler(self.socket.fileno(), self._handle_events,
                                     self._state)

    def connect(self, address, callback=None):
        """Connects the socket to a remote address without blocking.

        May only be called if the socket passed to the constructor was
        not previously connected.  The address parameter is in the
        same format as for socket.connect, i.e. a (host, port) tuple.
        If callback is specified, it will be called when the
        connection is completed.

        Note that it is safe to call IOStream.write while the
        connection is pending, in which case the data will be written
        as soon as the connection is ready.  Calling IOStream read
        methods before the socket is connected works on some platforms
        but is non-portable.
        """
        self._connecting = True
        try:
            self.socket.connect(address)
        except socket.error, e:
            # In non-blocking mode connect() always raises an exception
            if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
                raise
        self._connect_callback = stack_context.wrap(callback)
        self._add_io_state(self.io_loop.WRITE)
Exemple #55
0
 def add_callback(self, callback, *args, **kwargs):
     self.reactor.callFromThread(
         self._run_callback,
         functools.partial(wrap(callback), *args, **kwargs))
 def library_function(callback):
     # capture the caller's context before introducing our own
     callback = wrap(callback)
     with StackContext(functools.partial(self.context, 'library')):
         self.io_loop.add_callback(
             functools.partial(library_inner_callback, callback))
Exemple #57
0
 def add_handler(self, fd, handler, events):
     fd, obj = self.split_fd(fd)
     self._handlers[fd] = (obj, stack_context.wrap(handler))
     self._impl.register(fd, events | self.ERROR)
 def add_handler(self, fd, handler, events):
     """Registers the given handler to receive the given events for fd."""
     self._handlers[fd] = stack_context.wrap(handler)
     self._impl.register(fd, events | self.ERROR)
Exemple #59
0
 def add_timeout(self, deadline, callback):
     timeout = _Timeout(deadline, stack_context.wrap(callback), self)
     heapq.heappush(self._timeouts, timeout)
     return timeout
Exemple #60
0
class IOStream(object):
    r"""A utility class to write to and read from a non-blocking socket.

    We support a non-blocking ``write()`` and a family of ``read_*()`` methods.
    All of the methods take callbacks (since writing and reading are
    non-blocking and asynchronous).

    The socket parameter may either be connected or unconnected.  For
    server operations the socket is the result of calling socket.accept().
    For client operations the socket is created with socket.socket(),
    and may either be connected before passing it to the IOStream or
    connected with IOStream.connect.

    A very simple (and broken) HTTP client using this class::

        from tornado import ioloop
        from tornado import iostream
        import socket

        def send_request():
            stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n")
            stream.read_until("\r\n\r\n", on_headers)

        def on_headers(data):
            headers = {}
            for line in data.split("\r\n"):
               parts = line.split(":")
               if len(parts) == 2:
                   headers[parts[0].strip()] = parts[1].strip()
            stream.read_bytes(int(headers["Content-Length"]), on_body)

        def on_body(data):
            print data
            stream.close()
            ioloop.IOLoop.instance().stop()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        stream = iostream.IOStream(s)
        stream.connect(("friendfeed.com", 80), send_request)
        ioloop.IOLoop.instance().start()

    """
    def __init__(self,
                 socket,
                 io_loop=None,
                 max_buffer_size=104857600,
                 read_chunk_size=4096):
        self.socket = socket
        self.socket.setblocking(False)
        self.io_loop = io_loop or ioloop.IOLoop.instance()
        self.max_buffer_size = max_buffer_size
        self.read_chunk_size = read_chunk_size
        self._read_buffer = collections.deque()
        self._write_buffer = collections.deque()
        self._read_buffer_size = 0
        self._write_buffer_frozen = False
        self._read_delimiter = None
        self._read_regex = None
        self._read_bytes = None
        self._read_until_close = False
        self._read_callback = None
        self._streaming_callback = None
        self._write_callback = None
        self._close_callback = None
        self._connect_callback = None
        self._connecting = False
        self._state = None
        self._pending_callbacks = 0

    def connect(self, address, callback=None):
        """Connects the socket to a remote address without blocking.

        May only be called if the socket passed to the constructor was
        not previously connected.  The address parameter is in the
        same format as for socket.connect, i.e. a (host, port) tuple.
        If callback is specified, it will be called when the
        connection is completed.

        Note that it is safe to call IOStream.write while the
        connection is pending, in which case the data will be written
        as soon as the connection is ready.  Calling IOStream read
        methods before the socket is connected works on some platforms
        but is non-portable.
        """
        self._connecting = True
        try:
            self.socket.connect(address)
        except socket.error, e:
            # In non-blocking mode we expect connect() to raise an
            # exception with EINPROGRESS or EWOULDBLOCK.
            #
            # On freebsd, other errors such as ECONNREFUSED may be
            # returned immediately when attempting to connect to
            # localhost, so handle them the same way as an error
            # reported later in _handle_connect.
            if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
                logging.warning("Connect error on fd %d: %s",
                                self.socket.fileno(), e)
                self.close()
                return
        self._connect_callback = stack_context.wrap(callback)
        self._add_io_state(self.io_loop.WRITE)