class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. It does not currently implement all applicable parts of the HTTP specification, but it does enough to work with major web service APIs. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections are not reused, and callers cannot select the network interface to be used. """ def initialize(self, io_loop, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ super(SimpleAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) def close(self): super(SimpleAsyncHTTPClient, self).close() if self.own_resolver: self.resolver.close() def fetch_impl(self, request, callback): gen_log.warning("now we use the simple http client one") key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key)) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( len(self.active), len(self.queue))) def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: key, request, callback = self.queue.popleft() if key not in self.waiting: continue self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) def _handle_request(self, request, release_callback, final_callback): _HTTPConnection(self.io_loop, self, request, release_callback, final_callback, self.max_buffer_size, self.resolver) def _release_fetch(self, key): del self.active[key] self._process_queue() def _remove_timeout(self, key): if key in self.waiting: request, callback, timeout_handle = self.waiting[key] if timeout_handle is not None: self.io_loop.remove_timeout(timeout_handle) del self.waiting[key] def _on_timeout(self, key): request, callback, timeout_handle = self.waiting[key] self.queue.remove((key, request, callback)) timeout_response = HTTPResponse( request, 599, error=HTTPError(599, "Timeout"), request_time=self.io_loop.time() - request.start_time) self.io_loop.add_callback(callback, timeout_response) del self.waiting[key]
class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections are not reused, and callers cannot select the network interface to be used. """ def initialize(self, io_loop, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, max_header_size=None, max_body_size=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. ``force_instance=True`` may be used to suppress this behavior. Note that because of this implicit reuse, unless ``force_instance`` is used, only the first call to the constructor actually uses its arguments. It is recommended to use the ``configure`` method instead of the constructor to ensure that arguments take effect. ``max_clients`` is the number of concurrent requests that can be in progress; when this limit is reached additional requests will be queued. Note that time spent waiting in this queue still counts against the ``request_timeout``. ``hostname_mapping`` is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like ``/etc/hosts`` is not possible or desirable (e.g. in unittests). ``max_buffer_size`` (default 100MB) is the number of bytes that can be read into memory at once. ``max_body_size`` (defaults to ``max_buffer_size``) is the largest response body that the client will accept. Without a ``streaming_callback``, the smaller of these two limits applies; with a ``streaming_callback`` only ``max_body_size`` does. .. versionchanged:: 4.2 Added the ``max_body_size`` argument. """ super(SimpleAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop) def close(self): super(SimpleAsyncHTTPClient, self).close() if self.own_resolver: self.resolver.close() self.tcp_client.close() def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key, "in request queue")) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue))) def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: key, request, callback = self.queue.popleft() if key not in self.waiting: continue self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) def _connection_class(self): return _HTTPConnection def _handle_request(self, request, release_callback, final_callback): self._connection_class()(self.io_loop, self, request, release_callback, final_callback, self.max_buffer_size, self.tcp_client, self.max_header_size, self.max_body_size) def _release_fetch(self, key): del self.active[key] self._process_queue() def _remove_timeout(self, key): if key in self.waiting: request, callback, timeout_handle = self.waiting[key] if timeout_handle is not None: self.io_loop.remove_timeout(timeout_handle) del self.waiting[key] def _on_timeout(self, key, info=None): """Timeout callback of request. Construct a timeout HTTPResponse when a timeout occurs. :arg object key: A simple object to mark the request. :info string key: More detailed timeout information. """ request, callback, timeout_handle = self.waiting[key] self.queue.remove((key, request, callback)) error_message = "Timeout {0}".format(info) if info else "Timeout" timeout_response = HTTPResponse(request, 599, error=HTTPError(599, error_message), request_time=self.io_loop.time() - request.start_time) self.io_loop.add_callback(callback, timeout_response) del self.waiting[key]
class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. It does not currently implement all applicable parts of the HTTP specification, but it does enough to work with major web service APIs. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections are not reused, and callers cannot select the network interface to be used. """ def initialize(self, io_loop, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, max_header_size=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ super(SimpleAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop) def close(self): super(SimpleAsyncHTTPClient, self).close() if self.own_resolver: self.resolver.close() self.tcp_client.close() def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key)) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue))) def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: key, request, callback = self.queue.popleft() if key not in self.waiting: continue self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) def _handle_request(self, request, release_callback, final_callback): _HTTPConnection(self.io_loop, self, request, release_callback, final_callback, self.max_buffer_size, self.tcp_client, self.max_header_size) def _release_fetch(self, key): del self.active[key] self._process_queue() def _remove_timeout(self, key): if key in self.waiting: request, callback, timeout_handle = self.waiting[key] if timeout_handle is not None: self.io_loop.remove_timeout(timeout_handle) del self.waiting[key] def _on_timeout(self, key): request, callback, timeout_handle = self.waiting[key] self.queue.remove((key, request, callback)) timeout_response = HTTPResponse(request, 599, error=HTTPError(599, "Timeout"), request_time=self.io_loop.time() - request.start_time) self.io_loop.add_callback(callback, timeout_response) del self.waiting[key]
class SimpleAsyncHTTPClient(AsyncHTTPClient): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections are not reused, and callers cannot select the network interface to be used. """ def initialize(self, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, max_header_size=None, max_body_size=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. ``force_instance=True`` may be used to suppress this behavior. Note that because of this implicit reuse, unless ``force_instance`` is used, only the first call to the constructor actually uses its arguments. It is recommended to use the ``configure`` method instead of the constructor to ensure that arguments take effect. ``max_clients`` is the number of concurrent requests that can be in progress; when this limit is reached additional requests will be queued. Note that time spent waiting in this queue still counts against the ``request_timeout``. ``hostname_mapping`` is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like ``/etc/hosts`` is not possible or desirable (e.g. in unittests). ``max_buffer_size`` (default 100MB) is the number of bytes that can be read into memory at once. ``max_body_size`` (defaults to ``max_buffer_size``) is the largest response body that the client will accept. Without a ``streaming_callback``, the smaller of these two limits applies; with a ``streaming_callback`` only ``max_body_size`` does. .. versionchanged:: 4.2 Added the ``max_body_size`` argument. """ super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver() self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = TCPClient(resolver=self.resolver) def close(self): super(SimpleAsyncHTTPClient, self).close() if self.own_resolver: self.resolver.close() self.tcp_client.close() def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key, "in request queue")) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( len(self.active), len(self.queue))) def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: key, request, callback = self.queue.popleft() if key not in self.waiting: continue self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) def _connection_class(self): return _HTTPConnection def _handle_request(self, request, release_callback, final_callback): self._connection_class()( self, request, release_callback, final_callback, self.max_buffer_size, self.tcp_client, self.max_header_size, self.max_body_size) def _release_fetch(self, key): del self.active[key] self._process_queue() def _remove_timeout(self, key): if key in self.waiting: request, callback, timeout_handle = self.waiting[key] if timeout_handle is not None: self.io_loop.remove_timeout(timeout_handle) del self.waiting[key] def _on_timeout(self, key, info=None): """Timeout callback of request. Construct a timeout HTTPResponse when a timeout occurs. :arg object key: A simple object to mark the request. :info string key: More detailed timeout information. """ request, callback, timeout_handle = self.waiting[key] self.queue.remove((key, request, callback)) error_message = "Timeout {0}".format(info) if info else "Timeout" timeout_response = HTTPResponse( request, 599, error=HTTPTimeoutError(error_message), request_time=self.io_loop.time() - request.start_time) self.io_loop.add_callback(callback, timeout_response) del self.waiting[key]
class SimpleKeepAliveHTTPClient(object): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. It does not currently implement all applicable parts of the HTTP specification, but it does enough to work with major web service APIs. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections are not reused, and callers cannot select the network interface to be used. """ def __init__(self, io_loop, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, idle_timeout=30.0): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ self.io_loop = io_loop self.queue = collections.deque() self.active = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=self.io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.defaults = dict(HTTPRequest._DEFAULTS) if defaults is not None: self.defaults.update(defaults) self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver) self.idle_timeout = idle_timeout self._idle_timeout_callback = None self.logger = logging.getLogger(self.__class__.__name__) def fetch(self, request, callback=None, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. The ``Future`` wil raise an `HTTPError` if the request returned a non-200 response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse(request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if response.error: future.set_exception(response.error) else: future.set_result(response) self.fetch_impl(request, handle_response) return future def close(self): super(SimpleKeepAliveHTTPClient, self).close() if self.own_resolver: self.resolver.close() def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) self._process_queue() def _process_queue(self): with stack_context.NullContext(): if len(self.queue) and len(self.active) < 1: key, request, callback = self.queue.popleft() self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) if len(self.queue) == 0 and len(self.active) == 0: now = self.io_loop.time() self._idle_timeout_callback = self.io_loop.add_timeout( now + self.idle_timeout, stack_context.wrap(self._on_idle_timeout)) else: if self._idle_timeout_callback: self.io_loop.remove_timeout(self._idle_timeout_callback) self._idle_timeout_callback = None def _handle_request(self, request, release_callback, final_callback): if self.connection.final_callback and not self.connection.is_support_keepalive: self.logger.info( 'old connection does not support KA. creating new connection') self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver) self.connection.add_request(request, release_callback, final_callback) def _release_fetch(self, key): del self.active[key] self._process_queue() def _on_idle_timeout(self): msg = 'idle timeout' if hasattr(self, 'res_id'): msg += ' for {}'.format(self.res_id) self.logger.info(msg) self.connection.disconnect() def __len__(self): return len(self.queue) + len(self.active)
class SimpleKeepAliveHTTPClient(object): """Non-blocking HTTP client with no external dependencies. This class implements an HTTP 1.1 client on top of Tornado's IOStreams. It does not currently implement all applicable parts of the HTTP specification, but it does enough to work with major web service APIs. Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections are not reused, and callers cannot select the network interface to be used. """ def __init__(self, io_loop, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, idle_timeout=30.0): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ self.io_loop = io_loop self.queue = collections.deque() self.active = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=self.io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.defaults = dict(HTTPRequest._DEFAULTS) if defaults is not None: self.defaults.update(defaults) self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver) self.idle_timeout = idle_timeout self._idle_timeout_callback = None self.logger = logging.getLogger(self.__class__.__name__) def fetch(self, request, callback=None, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. The ``Future`` wil raise an `HTTPError` if the request returned a non-200 response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if response.error: future.set_exception(response.error) else: future.set_result(response) self.fetch_impl(request, handle_response) return future def close(self): super(SimpleKeepAliveHTTPClient, self).close() if self.own_resolver: self.resolver.close() def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) self._process_queue() def _process_queue(self): with stack_context.NullContext(): if len (self.queue) and len(self.active) < 1: key, request, callback = self.queue.popleft() self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) if len(self.queue) == 0 and len(self.active) == 0: now = self.io_loop.time() self._idle_timeout_callback = self.io_loop.add_timeout( now + self.idle_timeout, stack_context.wrap(self._on_idle_timeout) ) else: if self._idle_timeout_callback: self.io_loop.remove_timeout(self._idle_timeout_callback) self._idle_timeout_callback = None def _handle_request(self, request, release_callback, final_callback): if self.connection.final_callback and not self.connection.is_support_keepalive: self.logger.info('old connection does not support KA. creating new connection') self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver) self.connection.add_request(request, release_callback, final_callback) def _release_fetch(self, key): del self.active[key] self._process_queue() def _on_idle_timeout(self): msg = 'idle timeout' if hasattr(self, 'res_id'): msg += ' for {}'.format(self.res_id) self.logger.info(msg) self.connection.disconnect() def __len__(self): return len(self.queue) + len(self.active)
class HTTPAdapter(BaseAdapter): """The built-in HTTP Adapter for BaseIOStream. Provides a general-case interface for trip sessions to contact HTTP urls by implementing the Transport Adapter interface. This class will usually be created by the :class:`Session <Session>` class under the covers. :param max_retries: The maximum number of retries each connection should attempt. Note, this applies only to failed DNS lookups, socket connections and connection timeouts, never to requests where data has made it to the server. By default, Requests does not retry failed connections. #TODO: If you need granular control over the conditions under which we retry a request, import urllib3's ``Retry`` class and pass that instead. Usage:: >>> import trip >>> s = trip.Session() >>> a = trip.adapters.HTTPAdapter(hostname_mapping='/etc/hosts') >>> s.mount('http://', a) """ def __init__(self, io_loop=None, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, max_header_size=None, max_body_size=None): super(HTTPAdapter, self).__init__() self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size self.io_loop = io_loop or IOLoop.current() self.resolver = Resolver() if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = TCPClient(resolver=self.resolver) def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): """Sends Request object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: trip.adapters.MessageDelegate """ future = Future() def callback(response): if isinstance(response, Exception): future.set_exception(response) else: future.set_result(response) key = object() request = (request, stream, timeout, verify, cert, proxies) self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(parse_timeout(timeout)), functools.partial(self._on_timeout, key, 'in request queue')) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug('max_clients limit reached, request queued. ' '%d active, %d queued requests.' % (len(self.active), len(self.queue))) return future def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: key, request, callback = self.queue.popleft() if key not in self.waiting: continue self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) def _handle_request(self, request, release_callback, final_callback): request, stream, timeout, verify, cert, proxies = request HTTPConnection(request, release_callback, final_callback, self.tcp_client, self.max_buffer_size, self.max_header_size, self.max_body_size).send(stream, timeout, verify, cert, proxies) def _release_fetch(self, key): del self.active[key] self._process_queue() def _remove_timeout(self, key): if key in self.waiting: request, callback, timeout_handle = self.waiting[key] if timeout_handle is not None: self.io_loop.remove_timeout(timeout_handle) del self.waiting[key] def _on_timeout(self, key, info=None): """Timeout callback of request. Construct a timeout HTTPResponse when a timeout occurs. :arg object key: A simple object to mark the request. :info string key: More detailed timeout information. """ request, callback, timeout_handle = self.waiting[key] self.queue.remove((key, request, callback)) error_message = 'Timeout {0}'.format(info) if info else 'Timeout' self.io_loop.add_callback(callback, Timeout(599, err_message)) del self.waiting[key] def close(self): """Cleans up adapter specific items.""" self.resolver.close() self.tcp_client.close()
class SimpleAsyncUDPClient(AsyncUDPClient): def initialize( # type: ignore self, max_clients: int = 10, hostname_mapping: Dict[str, str] = None, max_buffer_size: int = 104857600, resolver: Resolver = None, defaults: Dict[str, Any] = None, max_header_size: int = None, max_body_size: int = None, ) -> None: super(SimpleAsyncUDPClient, self).initialize(defaults=defaults) self.max_clients = max_clients self.queue = ( collections.deque() ) # type: Deque[Tuple[object, UDPRequest, Callable[[UDPResponse], None]]] self.active = ( {} ) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None]]] self.waiting = ( {} ) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None], object]] self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver() self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = UDPClient(resolver=self.resolver) def close(self) -> None: super(SimpleAsyncUDPClient, self).close() if self.own_resolver: self.resolver.close() self.tcp_client.close() def fetch_impl(self, request: UDPRequest, callback: Callable[[UDPResponse], None]) -> None: key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: assert request.connect_timeout is not None assert request.request_timeout is not None timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key, "in request queue"), ) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue))) def _process_queue(self) -> None: while self.queue and len(self.active) < self.max_clients: key, request, callback = self.queue.popleft() if key not in self.waiting: continue self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) def _connection_class(self) -> type: return _UDPConnection def _handle_request( self, request: UDPRequest, release_callback: Callable[[], None], final_callback: Callable[[UDPResponse], None], ) -> None: self._connection_class()( self, request, release_callback, final_callback, self.max_buffer_size, self.tcp_client, self.max_header_size, self.max_body_size, ) def _release_fetch(self, key: object) -> None: del self.active[key] self._process_queue() def _remove_timeout(self, key: object) -> None: if key in self.waiting: request, callback, timeout_handle = self.waiting[key] if timeout_handle is not None: self.io_loop.remove_timeout(timeout_handle) del self.waiting[key] def _on_timeout(self, key: object, info: str = None) -> None: """Timeout callback of request. Construct a timeout HTTPResponse when a timeout occurs. :arg object key: A simple object to mark the request. :info string key: More detailed timeout information. """ request, callback, timeout_handle = self.waiting[key] self.queue.remove((key, request, callback)) error_message = "Timeout {0}".format(info) if info else "Timeout" timeout_response = UDPResponse( request, 599, #error=HTTPTimeoutError(error_message), request_time=self.io_loop.time() - request.start_time, ) self.io_loop.add_callback(callback, timeout_response) del self.waiting[key]