def initialize(
self, io_loop, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None
):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop, defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping)
def initialize(self,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
class OverrideResolverTest(AsyncTestCase, _ResolverTestMixin):
def setUp(self):
super(OverrideResolverTest, self).setUp()
mapping = {
('google.com', 80): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET6):
('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80)
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
def test_resolve_multiaddr(self):
self.resolver.resolve('google.com',
80,
socket.AF_INET,
callback=self.stop)
result = self.wait()
self.assertIn((socket.AF_INET, ('1.2.3.4', 80)), result)
self.resolver.resolve('google.com',
80,
socket.AF_INET6,
callback=self.stop)
result = self.wait()
self.assertIn(
(socket.AF_INET6, ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80, 0, 0)),
result)
def setUp(self):
super(OverrideResolverTest, self).setUp()
mapping = {
('google.com', 80): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET6): ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80)
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
def initialize(self, io_loop=None, max_clients=10,
max_simultaneous_connections=None,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None, max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def setUp(self):
super().setUp()
mapping = {
("google.com", 80): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET6): (
"2a02:6b8:7c:40c:c51e:495f:e23a:3",
80,
),
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
def __init__(self,
io_loop,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
idle_timeout=30.0):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
self.io_loop = io_loop
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=self.io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.defaults = dict(HTTPRequest._DEFAULTS)
if defaults is not None:
self.defaults.update(defaults)
self.connection = KeepAliveHTTPConnection(self.io_loop, self,
self.max_buffer_size,
self.resolver)
self.idle_timeout = idle_timeout
self._idle_timeout_callback = None
self.logger = logging.getLogger(self.__class__.__name__)
def initialize(self,
io_loop,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None):
"""Creates a AsyncHTTPClient.
# 一个IOLoop实例只有一个AsyncHTTPClient实例
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients # 最大连接数
self.queue = collections.deque() # 来一个小小的队列
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
# 这里估计是一个解析DNS的配置，先不理会
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
# 这里来了一个tcp的连接
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def initialize(self, io_loop, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
self.resolver = resolver or Resolver(io_loop=io_loop)
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
class OverrideResolverTest(AsyncTestCase, _ResolverTestMixin):
def setUp(self):
super(OverrideResolverTest, self).setUp()
mapping = {
('google.com', 80): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET6): ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80)
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
@gen_test
def test_resolve_multiaddr(self):
result = yield self.resolver.resolve('google.com', 80, socket.AF_INET)
self.assertIn((socket.AF_INET, ('1.2.3.4', 80)), result)
result = yield self.resolver.resolve('google.com', 80, socket.AF_INET6)
self.assertIn((socket.AF_INET6, ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80, 0, 0)), result)
def setUp(self):
super(OverrideResolverTest, self).setUp()
mapping = {
('google.com', 80): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET6): ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80)
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
def __init__(self, io_loop,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None, idle_timeout=30.0):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
self.io_loop = io_loop
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=self.io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.defaults = dict(HTTPRequest._DEFAULTS)
if defaults is not None:
self.defaults.update(defaults)
self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver)
self.idle_timeout = idle_timeout
self._idle_timeout_callback = None
self.logger = logging.getLogger(self.__class__.__name__)
def setUp(self):
super(OverrideResolverTest, self).setUp()
mapping = {
("google.com", 80): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET6): (
"2a02:6b8:7c:40c:c51e:495f:e23a:3",
80,
),
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
class OverrideResolverTest(AsyncTestCase, _ResolverTestMixin):
def setUp(self):
super().setUp()
mapping = {
("google.com", 80): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET6): (
"2a02:6b8:7c:40c:c51e:495f:e23a:3",
80,
),
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
@gen_test
def test_resolve_multiaddr(self):
result = yield self.resolver.resolve("google.com", 80, socket.AF_INET)
self.assertIn((socket.AF_INET, ("1.2.3.4", 80)), result)
result = yield self.resolver.resolve("google.com", 80, socket.AF_INET6)
self.assertIn(
(socket.AF_INET6, ("2a02:6b8:7c:40c:c51e:495f:e23a:3", 80, 0, 0)),
result)
def initialize( # type: ignore
self,
max_clients: int = 10,
hostname_mapping: Dict[str, str] = None,
max_buffer_size: int = 104857600,
resolver: Resolver = None,
defaults: Dict[str, Any] = None,
max_header_size: int = None,
max_body_size: int = None,
) -> None:
super(SimpleAsyncUDPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = (
collections.deque()
) # type: Deque[Tuple[object, UDPRequest, Callable[[UDPResponse], None]]]
self.active = (
{}
) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None]]]
self.waiting = (
{}
) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None], object]]
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = UDPClient(resolver=self.resolver)
def __init__(self,
io_loop=None,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
max_header_size=None,
max_body_size=None):
super(HTTPAdapter, self).__init__()
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
self.io_loop = io_loop or IOLoop.current()
self.resolver = Resolver()
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
def do_test(ips):
print len(ips)
rss = []
rs = []
old = 0
ips = list(set(ips))
for i, item in enumerate(ips):
resolver = OverrideResolver(
Resolver(),
mapping={'clarkhillgo1.appspot.com': item.replace('\n', '')})
request = HTTPRequest('https://clarkhillgo1.appspot.com',
validate_cert=False)
client = SimpleAsyncHTTPClient(resolver=resolver, force_instance=True)
rs.append((client.fetch(request), item, client))
while len(rs) > 0:
if old != len(rs):
print time.time(), len(rs)
old = len(rs)
for f in rs:
if f[0].done():
rs = [_ for _ in rs if _ != f]
if f[0].exception():
f[2].close()
# print 'exception: ', f[0].exception()
else:
body = f[0].result().body
if 'GoAgent' in body:
print body, f[1]
rss.append(f[1])
f[2].close()
yield gen.sleep(0.5)
if rss:
with open('./all_good_ip.txt', 'a') as f:
for r in rss:
f.writelines(r + '\n')
class SimpleAsyncHTTPClient(AsyncHTTPClient):
"""Non-blocking HTTP client with no external dependencies.
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
It does not currently implement all applicable parts of the HTTP
specification, but it does enough to work with major web service APIs.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
are not reused, and callers cannot select the network interface to be
used.
"""
def initialize(self, io_loop, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
def close(self):
super(SimpleAsyncHTTPClient, self).close()
if self.own_resolver:
self.resolver.close()
def fetch_impl(self, request, callback):
gen_log.warning("now we use the simple http client one")
key = object()
self.queue.append((key, request, callback))
if not len(self.active) < self.max_clients:
timeout_handle = self.io_loop.add_timeout(
self.io_loop.time() + min(request.connect_timeout,
request.request_timeout),
functools.partial(self._on_timeout, key))
else:
timeout_handle = None
self.waiting[key] = (request, callback, timeout_handle)
self._process_queue()
if self.queue:
gen_log.debug("max_clients limit reached, request queued. "
"%d active, %d queued requests." % (
len(self.active), len(self.queue)))
def _process_queue(self):
with stack_context.NullContext():
while self.queue and len(self.active) < self.max_clients:
key, request, callback = self.queue.popleft()
if key not in self.waiting:
continue
self._remove_timeout(key)
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _handle_request(self, request, release_callback, final_callback):
_HTTPConnection(self.io_loop, self, request, release_callback,
final_callback, self.max_buffer_size, self.resolver)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _remove_timeout(self, key):
if key in self.waiting:
request, callback, timeout_handle = self.waiting[key]
if timeout_handle is not None:
self.io_loop.remove_timeout(timeout_handle)
del self.waiting[key]
def _on_timeout(self, key):
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
timeout_response = HTTPResponse(
request, 599, error=HTTPError(599, "Timeout"),
request_time=self.io_loop.time() - request.start_time)
self.io_loop.add_callback(callback, timeout_response)
del self.waiting[key]
class SimpleAsyncHTTPClient(AsyncHTTPClient):
"""Non-blocking HTTP client with no external dependencies.
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
It does not currently implement all applicable parts of the HTTP
specification, but it does enough to work with major web service APIs.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
are not reused, and callers cannot select the network interface to be
used.
"""
def initialize(self,
io_loop,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def close(self):
super(SimpleAsyncHTTPClient, self).close()
if self.own_resolver:
self.resolver.close()
self.tcp_client.close()
def fetch_impl(self, request, callback):
key = object()
self.queue.append((key, request, callback))
if not len(self.active) < self.max_clients:
timeout_handle = self.io_loop.add_timeout(
self.io_loop.time() +
min(request.connect_timeout, request.request_timeout),
functools.partial(self._on_timeout, key))
else:
timeout_handle = None
self.waiting[key] = (request, callback, timeout_handle)
self._process_queue()
if self.queue:
gen_log.debug("max_clients limit reached, request queued. "
"%d active, %d queued requests." %
(len(self.active), len(self.queue)))
def _process_queue(self):
with stack_context.NullContext():
while self.queue and len(self.active) < self.max_clients:
key, request, callback = self.queue.popleft()
if key not in self.waiting:
continue
self._remove_timeout(key)
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _handle_request(self, request, release_callback, final_callback):
_HTTPConnection(self.io_loop, self, request, release_callback,
final_callback, self.max_buffer_size, self.tcp_client,
self.max_header_size)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _remove_timeout(self, key):
if key in self.waiting:
request, callback, timeout_handle = self.waiting[key]
if timeout_handle is not None:
self.io_loop.remove_timeout(timeout_handle)
del self.waiting[key]
def _on_timeout(self, key):
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
timeout_response = HTTPResponse(request,
599,
error=HTTPError(599, "Timeout"),
request_time=self.io_loop.time() -
request.start_time)
self.io_loop.add_callback(callback, timeout_response)
del self.waiting[key]
class SimpleAsyncHTTPClient(AsyncHTTPClient):
"""Non-blocking HTTP client with no external dependencies.
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
are not reused, and callers cannot select the network interface to be
used.
"""
def initialize(self,
io_loop,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def close(self):
super(SimpleAsyncHTTPClient, self).close()
if self.own_resolver:
self.resolver.close()
self.tcp_client.close()
def fetch_impl(self, request, callback):
key = object()
self.queue.append((key, request, callback))
if not len(self.active) < self.max_clients:
timeout_handle = self.io_loop.add_timeout(
self.io_loop.time() +
min(request.connect_timeout, request.request_timeout),
functools.partial(self._on_timeout, key, "in request queue"))
else:
timeout_handle = None
self.waiting[key] = (request, callback, timeout_handle)
self._process_queue()
if self.queue:
gen_log.debug("max_clients limit reached, request queued. "
"%d active, %d queued requests." %
(len(self.active), len(self.queue)))
def _process_queue(self):
with stack_context.NullContext():
while self.queue and len(self.active) < self.max_clients:
key, request, callback = self.queue.popleft()
if key not in self.waiting:
continue
self._remove_timeout(key)
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _connection_class(self):
return _HTTPConnection
def _handle_request(self, request, release_callback, final_callback):
self._connection_class()(self.io_loop, self, request, release_callback,
final_callback, self.max_buffer_size,
self.tcp_client, self.max_header_size,
self.max_body_size)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _remove_timeout(self, key):
if key in self.waiting:
request, callback, timeout_handle = self.waiting[key]
if timeout_handle is not None:
self.io_loop.remove_timeout(timeout_handle)
del self.waiting[key]
def _on_timeout(self, key, info=None):
"""Timeout callback of request.
Construct a timeout HTTPResponse when a timeout occurs.
:arg object key: A simple object to mark the request.
:info string key: More detailed timeout information.
"""
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
error_message = "Timeout {0}".format(info) if info else "Timeout"
timeout_response = HTTPResponse(request,
599,
error=HTTPError(599, error_message),
request_time=self.io_loop.time() -
request.start_time)
self.io_loop.add_callback(callback, timeout_response)
del self.waiting[key]
class SimpleAsyncUDPClient(AsyncUDPClient):
def initialize( # type: ignore
self,
max_clients: int = 10,
hostname_mapping: Dict[str, str] = None,
max_buffer_size: int = 104857600,
resolver: Resolver = None,
defaults: Dict[str, Any] = None,
max_header_size: int = None,
max_body_size: int = None,
) -> None:
super(SimpleAsyncUDPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = (
collections.deque()
) # type: Deque[Tuple[object, UDPRequest, Callable[[UDPResponse], None]]]
self.active = (
{}
) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None]]]
self.waiting = (
{}
) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None], object]]
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = UDPClient(resolver=self.resolver)
def close(self) -> None:
super(SimpleAsyncUDPClient, self).close()
if self.own_resolver:
self.resolver.close()
self.tcp_client.close()
def fetch_impl(self, request: UDPRequest,
callback: Callable[[UDPResponse], None]) -> None:
key = object()
self.queue.append((key, request, callback))
if not len(self.active) < self.max_clients:
assert request.connect_timeout is not None
assert request.request_timeout is not None
timeout_handle = self.io_loop.add_timeout(
self.io_loop.time() +
min(request.connect_timeout, request.request_timeout),
functools.partial(self._on_timeout, key, "in request queue"),
)
else:
timeout_handle = None
self.waiting[key] = (request, callback, timeout_handle)
self._process_queue()
if self.queue:
gen_log.debug("max_clients limit reached, request queued. "
"%d active, %d queued requests." %
(len(self.active), len(self.queue)))
def _process_queue(self) -> None:
while self.queue and len(self.active) < self.max_clients:
key, request, callback = self.queue.popleft()
if key not in self.waiting:
continue
self._remove_timeout(key)
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _connection_class(self) -> type:
return _UDPConnection
def _handle_request(
self,
request: UDPRequest,
release_callback: Callable[[], None],
final_callback: Callable[[UDPResponse], None],
) -> None:
self._connection_class()(
self,
request,
release_callback,
final_callback,
self.max_buffer_size,
self.tcp_client,
self.max_header_size,
self.max_body_size,
)
def _release_fetch(self, key: object) -> None:
del self.active[key]
self._process_queue()
def _remove_timeout(self, key: object) -> None:
if key in self.waiting:
request, callback, timeout_handle = self.waiting[key]
if timeout_handle is not None:
self.io_loop.remove_timeout(timeout_handle)
del self.waiting[key]
def _on_timeout(self, key: object, info: str = None) -> None:
"""Timeout callback of request.
Construct a timeout HTTPResponse when a timeout occurs.
:arg object key: A simple object to mark the request.
:info string key: More detailed timeout information.
"""
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
error_message = "Timeout {0}".format(info) if info else "Timeout"
timeout_response = UDPResponse(
request,
599,
#error=HTTPTimeoutError(error_message),
request_time=self.io_loop.time() - request.start_time,
)
self.io_loop.add_callback(callback, timeout_response)
del self.waiting[key]
class HTTPAdapter(BaseAdapter):
"""The built-in HTTP Adapter for BaseIOStream.
Provides a general-case interface for trip sessions to contact HTTP urls
by implementing the Transport Adapter interface. This class will
usually be created by the :class:`Session <Session>` class under the
covers.
:param max_retries: The maximum number of retries each connection
should attempt. Note, this applies only to failed DNS lookups, socket
connections and connection timeouts, never to requests where data has
made it to the server. By default, Requests does not retry failed
connections.
#TODO: If you need granular control over the conditions under
which we retry a request, import urllib3's ``Retry`` class and pass
that instead.
Usage::
>>> import trip
>>> s = trip.Session()
>>> a = trip.adapters.HTTPAdapter(hostname_mapping='/etc/hosts')
>>> s.mount('http://', a)
"""
def __init__(self,
io_loop=None,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
max_header_size=None,
max_body_size=None):
super(HTTPAdapter, self).__init__()
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
self.io_loop = io_loop or IOLoop.current()
self.resolver = Resolver()
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
def send(self,
request,
stream=False,
timeout=None,
verify=True,
cert=None,
proxies=None):
"""Sends Request object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param verify: (optional) Whether to verify SSL certificates.
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: trip.adapters.MessageDelegate
"""
future = Future()
def callback(response):
if isinstance(response, Exception):
future.set_exception(response)
else:
future.set_result(response)
key = object()
request = (request, stream, timeout, verify, cert, proxies)
self.queue.append((key, request, callback))
if not len(self.active) < self.max_clients:
timeout_handle = self.io_loop.add_timeout(
self.io_loop.time() + min(parse_timeout(timeout)),
functools.partial(self._on_timeout, key, 'in request queue'))
else:
timeout_handle = None
self.waiting[key] = (request, callback, timeout_handle)
self._process_queue()
if self.queue:
gen_log.debug('max_clients limit reached, request queued. '
'%d active, %d queued requests.' %
(len(self.active), len(self.queue)))
return future
def _process_queue(self):
with stack_context.NullContext():
while self.queue and len(self.active) < self.max_clients:
key, request, callback = self.queue.popleft()
if key not in self.waiting:
continue
self._remove_timeout(key)
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _handle_request(self, request, release_callback, final_callback):
request, stream, timeout, verify, cert, proxies = request
HTTPConnection(request, release_callback, final_callback,
self.tcp_client, self.max_buffer_size,
self.max_header_size,
self.max_body_size).send(stream, timeout, verify, cert,
proxies)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _remove_timeout(self, key):
if key in self.waiting:
request, callback, timeout_handle = self.waiting[key]
if timeout_handle is not None:
self.io_loop.remove_timeout(timeout_handle)
del self.waiting[key]
def _on_timeout(self, key, info=None):
"""Timeout callback of request.
Construct a timeout HTTPResponse when a timeout occurs.
:arg object key: A simple object to mark the request.
:info string key: More detailed timeout information.
"""
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
error_message = 'Timeout {0}'.format(info) if info else 'Timeout'
self.io_loop.add_callback(callback, Timeout(599, err_message))
del self.waiting[key]
def close(self):
"""Cleans up adapter specific items."""
self.resolver.close()
self.tcp_client.close()
class SimpleAsyncHTTPClient(AsyncHTTPClient):
"""Non-blocking HTTP client with no external dependencies.
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
are not reused, and callers cannot select the network interface to be
used.
"""
def initialize(self, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None, max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
def close(self):
super(SimpleAsyncHTTPClient, self).close()
if self.own_resolver:
self.resolver.close()
self.tcp_client.close()
def fetch_impl(self, request, callback):
key = object()
self.queue.append((key, request, callback))
if not len(self.active) < self.max_clients:
timeout_handle = self.io_loop.add_timeout(
self.io_loop.time() + min(request.connect_timeout,
request.request_timeout),
functools.partial(self._on_timeout, key, "in request queue"))
else:
timeout_handle = None
self.waiting[key] = (request, callback, timeout_handle)
self._process_queue()
if self.queue:
gen_log.debug("max_clients limit reached, request queued. "
"%d active, %d queued requests." % (
len(self.active), len(self.queue)))
def _process_queue(self):
with stack_context.NullContext():
while self.queue and len(self.active) < self.max_clients:
key, request, callback = self.queue.popleft()
if key not in self.waiting:
continue
self._remove_timeout(key)
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
def _connection_class(self):
return _HTTPConnection
def _handle_request(self, request, release_callback, final_callback):
self._connection_class()(
self, request, release_callback,
final_callback, self.max_buffer_size, self.tcp_client,
self.max_header_size, self.max_body_size)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _remove_timeout(self, key):
if key in self.waiting:
request, callback, timeout_handle = self.waiting[key]
if timeout_handle is not None:
self.io_loop.remove_timeout(timeout_handle)
del self.waiting[key]
def _on_timeout(self, key, info=None):
"""Timeout callback of request.
Construct a timeout HTTPResponse when a timeout occurs.
:arg object key: A simple object to mark the request.
:info string key: More detailed timeout information.
"""
request, callback, timeout_handle = self.waiting[key]
self.queue.remove((key, request, callback))
error_message = "Timeout {0}".format(info) if info else "Timeout"
timeout_response = HTTPResponse(
request, 599, error=HTTPTimeoutError(error_message),
request_time=self.io_loop.time() - request.start_time)
self.io_loop.add_callback(callback, timeout_response)
del self.waiting[key]
class SimpleKeepAliveHTTPClient(object):
"""Non-blocking HTTP client with no external dependencies.
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
It does not currently implement all applicable parts of the HTTP
specification, but it does enough to work with major web service APIs.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
are not reused, and callers cannot select the network interface to be
used.
"""
def __init__(self,
io_loop,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
idle_timeout=30.0):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
self.io_loop = io_loop
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=self.io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.defaults = dict(HTTPRequest._DEFAULTS)
if defaults is not None:
self.defaults.update(defaults)
self.connection = KeepAliveHTTPConnection(self.io_loop, self,
self.max_buffer_size,
self.resolver)
self.idle_timeout = idle_timeout
self._idle_timeout_callback = None
self.logger = logging.getLogger(self.__class__.__name__)
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` wil raise an `HTTPError` if
the request returned a non-200 response code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(request,
599,
error=exc,
request_time=time.time() -
request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
else:
future.set_result(response)
self.fetch_impl(request, handle_response)
return future
def close(self):
super(SimpleKeepAliveHTTPClient, self).close()
if self.own_resolver:
self.resolver.close()
def fetch_impl(self, request, callback):
key = object()
self.queue.append((key, request, callback))
self._process_queue()
def _process_queue(self):
with stack_context.NullContext():
if len(self.queue) and len(self.active) < 1:
key, request, callback = self.queue.popleft()
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
if len(self.queue) == 0 and len(self.active) == 0:
now = self.io_loop.time()
self._idle_timeout_callback = self.io_loop.add_timeout(
now + self.idle_timeout,
stack_context.wrap(self._on_idle_timeout))
else:
if self._idle_timeout_callback:
self.io_loop.remove_timeout(self._idle_timeout_callback)
self._idle_timeout_callback = None
def _handle_request(self, request, release_callback, final_callback):
if self.connection.final_callback and not self.connection.is_support_keepalive:
self.logger.info(
'old connection does not support KA. creating new connection')
self.connection = KeepAliveHTTPConnection(self.io_loop, self,
self.max_buffer_size,
self.resolver)
self.connection.add_request(request, release_callback, final_callback)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _on_idle_timeout(self):
msg = 'idle timeout'
if hasattr(self, 'res_id'):
msg += ' for {}'.format(self.res_id)
self.logger.info(msg)
self.connection.disconnect()
def __len__(self):
return len(self.queue) + len(self.active)
from tornado.ioloop import IOLoop
from tornado.netutil import Resolver, OverrideResolver
from tornado.simple_httpclient import SimpleAsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import gen
request = HTTPRequest('http://www.baidu.com', validate_cert=False)
reslover = OverrideResolver(Resolver(),
mapping={"www.baidu.com": "111.13.101.208"})
client = SimpleAsyncHTTPClient(resolver=reslover)
@gen.coroutine
def get_response():
xxx = yield reslover.resolve('www.baidu.com', 80)
print xxx
rs = yield client.fetch(request)
print rs.body
IOLoop.current().run_sync(get_response)
class SimpleKeepAliveHTTPClient(object):
"""Non-blocking HTTP client with no external dependencies.
This class implements an HTTP 1.1 client on top of Tornado's IOStreams.
It does not currently implement all applicable parts of the HTTP
specification, but it does enough to work with major web service APIs.
Some features found in the curl-based AsyncHTTPClient are not yet
supported. In particular, proxies are not supported, connections
are not reused, and callers cannot select the network interface to be
used.
"""
def __init__(self, io_loop,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None, idle_timeout=30.0):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
self.io_loop = io_loop
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=self.io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.defaults = dict(HTTPRequest._DEFAULTS)
if defaults is not None:
self.defaults.update(defaults)
self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver)
self.idle_timeout = idle_timeout
self._idle_timeout_callback = None
self.logger = logging.getLogger(self.__class__.__name__)
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` wil raise an `HTTPError` if
the request returned a non-200 response code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(
request, 599, error=exc,
request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
else:
future.set_result(response)
self.fetch_impl(request, handle_response)
return future
def close(self):
super(SimpleKeepAliveHTTPClient, self).close()
if self.own_resolver:
self.resolver.close()
def fetch_impl(self, request, callback):
key = object()
self.queue.append((key, request, callback))
self._process_queue()
def _process_queue(self):
with stack_context.NullContext():
if len (self.queue) and len(self.active) < 1:
key, request, callback = self.queue.popleft()
self.active[key] = (request, callback)
release_callback = functools.partial(self._release_fetch, key)
self._handle_request(request, release_callback, callback)
if len(self.queue) == 0 and len(self.active) == 0:
now = self.io_loop.time()
self._idle_timeout_callback = self.io_loop.add_timeout(
now + self.idle_timeout,
stack_context.wrap(self._on_idle_timeout)
)
else:
if self._idle_timeout_callback:
self.io_loop.remove_timeout(self._idle_timeout_callback)
self._idle_timeout_callback = None
def _handle_request(self, request, release_callback, final_callback):
if self.connection.final_callback and not self.connection.is_support_keepalive:
self.logger.info('old connection does not support KA. creating new connection')
self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver)
self.connection.add_request(request, release_callback, final_callback)
def _release_fetch(self, key):
del self.active[key]
self._process_queue()
def _on_idle_timeout(self):
msg = 'idle timeout'
if hasattr(self, 'res_id'):
msg += ' for {}'.format(self.res_id)
self.logger.info(msg)
self.connection.disconnect()
def __len__(self):
return len(self.queue) + len(self.active)
