def initialize(self, io_loop, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
self.resolver = resolver or Resolver(io_loop=io_loop)
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
def initialize(self,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
def setUp(self):
super(OverrideResolverTest, self).setUp()
mapping = {
('google.com', 80): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET): ('1.2.3.4', 80),
('google.com', 80, socket.AF_INET6): ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80)
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
def setUp(self):
super().setUp()
mapping = {
("google.com", 80): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET): ("1.2.3.4", 80),
("google.com", 80, socket.AF_INET6): (
"2a02:6b8:7c:40c:c51e:495f:e23a:3",
80,
),
}
self.resolver = OverrideResolver(BlockingResolver(), mapping)
def __init__(self,
io_loop,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
idle_timeout=30.0):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
self.io_loop = io_loop
self.queue = collections.deque()
self.active = {}
self.max_buffer_size = max_buffer_size
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=self.io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.defaults = dict(HTTPRequest._DEFAULTS)
if defaults is not None:
self.defaults.update(defaults)
self.connection = KeepAliveHTTPConnection(self.io_loop, self,
self.max_buffer_size,
self.resolver)
self.idle_timeout = idle_timeout
self._idle_timeout_callback = None
self.logger = logging.getLogger(self.__class__.__name__)
def initialize(self,
io_loop,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None):
"""Creates a AsyncHTTPClient.
# 一个IOLoop实例只有一个AsyncHTTPClient实例
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients # 最大连接数
self.queue = collections.deque() # 来一个小小的队列
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
# 这里估计是一个解析DNS的配置，先不理会
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
# 这里来了一个tcp的连接
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def __init__(self, io_loop=None, hostname_mapping=None,
max_buffer_size=104857600, max_header_size=None,
max_body_size=None):
super(HTTPAdapter, self).__init__()
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
self.io_loop = io_loop or IOLoop.current()
self.resolver = Resolver()
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
def initialize( # type: ignore
self,
max_clients: int = 10,
hostname_mapping: Dict[str, str] = None,
max_buffer_size: int = 104857600,
resolver: Resolver = None,
defaults: Dict[str, Any] = None,
max_header_size: int = None,
max_body_size: int = None,
) -> None:
super(SimpleAsyncUDPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = (
collections.deque()
) # type: Deque[Tuple[object, UDPRequest, Callable[[UDPResponse], None]]]
self.active = (
{}
) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None]]]
self.waiting = (
{}
) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None], object]]
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = UDPClient(resolver=self.resolver)
def do_test(ips):
print len(ips)
rss = []
rs = []
old = 0
ips = list(set(ips))
for i, item in enumerate(ips):
resolver = OverrideResolver(
Resolver(),
mapping={'clarkhillgo1.appspot.com': item.replace('\n', '')})
request = HTTPRequest('https://clarkhillgo1.appspot.com',
validate_cert=False)
client = SimpleAsyncHTTPClient(resolver=resolver, force_instance=True)
rs.append((client.fetch(request), item, client))
while len(rs) > 0:
if old != len(rs):
print time.time(), len(rs)
old = len(rs)
for f in rs:
if f[0].done():
rs = [_ for _ in rs if _ != f]
if f[0].exception():
f[2].close()
# print 'exception: ', f[0].exception()
else:
body = f[0].result().body
if 'GoAgent' in body:
print body, f[1]
rss.append(f[1])
f[2].close()
yield gen.sleep(0.5)
if rss:
with open('./all_good_ip.txt', 'a') as f:
for r in rss:
f.writelines(r + '\n')
from tornado.ioloop import IOLoop
from tornado.netutil import Resolver, OverrideResolver
from tornado.simple_httpclient import SimpleAsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import gen
request = HTTPRequest('http://www.baidu.com', validate_cert=False)
reslover = OverrideResolver(Resolver(),
mapping={"www.baidu.com": "111.13.101.208"})
client = SimpleAsyncHTTPClient(resolver=reslover)
@gen.coroutine
def get_response():
xxx = yield reslover.resolve('www.baidu.com', 80)
print xxx
rs = yield client.fetch(request)
print rs.body
IOLoop.current().run_sync(get_response)
