def initialize(self, io_loop, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ super(SimpleAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.max_buffer_size = max_buffer_size self.resolver = resolver or Resolver(io_loop=io_loop) if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping)
def initialize(self, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, max_header_size=None, max_body_size=None): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. ``force_instance=True`` may be used to suppress this behavior. Note that because of this implicit reuse, unless ``force_instance`` is used, only the first call to the constructor actually uses its arguments. It is recommended to use the ``configure`` method instead of the constructor to ensure that arguments take effect. ``max_clients`` is the number of concurrent requests that can be in progress; when this limit is reached additional requests will be queued. Note that time spent waiting in this queue still counts against the ``request_timeout``. ``hostname_mapping`` is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like ``/etc/hosts`` is not possible or desirable (e.g. in unittests). ``max_buffer_size`` (default 100MB) is the number of bytes that can be read into memory at once. ``max_body_size`` (defaults to ``max_buffer_size``) is the largest response body that the client will accept. Without a ``streaming_callback``, the smaller of these two limits applies; with a ``streaming_callback`` only ``max_body_size`` does. .. versionchanged:: 4.2 Added the ``max_body_size`` argument. """ super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults) self.max_clients = max_clients self.queue = collections.deque() self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver() self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = TCPClient(resolver=self.resolver)
def setUp(self): super(OverrideResolverTest, self).setUp() mapping = { ('google.com', 80): ('1.2.3.4', 80), ('google.com', 80, socket.AF_INET): ('1.2.3.4', 80), ('google.com', 80, socket.AF_INET6): ('2a02:6b8:7c:40c:c51e:495f:e23a:3', 80) } self.resolver = OverrideResolver(BlockingResolver(), mapping)
def setUp(self): super().setUp() mapping = { ("google.com", 80): ("1.2.3.4", 80), ("google.com", 80, socket.AF_INET): ("1.2.3.4", 80), ("google.com", 80, socket.AF_INET6): ( "2a02:6b8:7c:40c:c51e:495f:e23a:3", 80, ), } self.resolver = OverrideResolver(BlockingResolver(), mapping)
def __init__(self, io_loop, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, idle_timeout=30.0): """Creates a AsyncHTTPClient. Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ self.io_loop = io_loop self.queue = collections.deque() self.active = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=self.io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.defaults = dict(HTTPRequest._DEFAULTS) if defaults is not None: self.defaults.update(defaults) self.connection = KeepAliveHTTPConnection(self.io_loop, self, self.max_buffer_size, self.resolver) self.idle_timeout = idle_timeout self._idle_timeout_callback = None self.logger = logging.getLogger(self.__class__.__name__)
def initialize(self, io_loop, max_clients=10, hostname_mapping=None, max_buffer_size=104857600, resolver=None, defaults=None, max_header_size=None): """Creates a AsyncHTTPClient. # 一个IOLoop实例只有一个AsyncHTTPClient实例 Only a single AsyncHTTPClient instance exists per IOLoop in order to provide limitations on the number of pending connections. force_instance=True may be used to suppress this behavior. max_clients is the number of concurrent requests that can be in progress. Note that this arguments are only used when the client is first created, and will be ignored when an existing client is reused. hostname_mapping is a dictionary mapping hostnames to IP addresses. It can be used to make local DNS changes when modifying system-wide settings like /etc/hosts is not possible or desirable (e.g. in unittests). max_buffer_size is the number of bytes that can be read by IOStream. It defaults to 100mb. """ super(SimpleAsyncHTTPClient, self).initialize(io_loop, defaults=defaults) self.max_clients = max_clients # 最大连接数 self.queue = collections.deque() # 来一个小小的队列 self.active = {} self.waiting = {} self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. # 这里估计是一个解析DNS的配置,先不理会 if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver(io_loop=io_loop) self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) # 这里来了一个tcp的连接 self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def __init__(self, io_loop=None, hostname_mapping=None, max_buffer_size=104857600, max_header_size=None, max_body_size=None): super(HTTPAdapter, self).__init__() self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size self.io_loop = io_loop or IOLoop.current() self.resolver = Resolver() if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = TCPClient(resolver=self.resolver)
def initialize( # type: ignore self, max_clients: int = 10, hostname_mapping: Dict[str, str] = None, max_buffer_size: int = 104857600, resolver: Resolver = None, defaults: Dict[str, Any] = None, max_header_size: int = None, max_body_size: int = None, ) -> None: super(SimpleAsyncUDPClient, self).initialize(defaults=defaults) self.max_clients = max_clients self.queue = ( collections.deque() ) # type: Deque[Tuple[object, UDPRequest, Callable[[UDPResponse], None]]] self.active = ( {} ) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None]]] self.waiting = ( {} ) # type: Dict[object, Tuple[UDPRequest, Callable[[UDPResponse], None], object]] self.max_buffer_size = max_buffer_size self.max_header_size = max_header_size self.max_body_size = max_body_size # TCPClient could create a Resolver for us, but we have to do it # ourselves to support hostname_mapping. if resolver: self.resolver = resolver self.own_resolver = False else: self.resolver = Resolver() self.own_resolver = True if hostname_mapping is not None: self.resolver = OverrideResolver(resolver=self.resolver, mapping=hostname_mapping) self.tcp_client = UDPClient(resolver=self.resolver)
def do_test(ips): print len(ips) rss = [] rs = [] old = 0 ips = list(set(ips)) for i, item in enumerate(ips): resolver = OverrideResolver( Resolver(), mapping={'clarkhillgo1.appspot.com': item.replace('\n', '')}) request = HTTPRequest('https://clarkhillgo1.appspot.com', validate_cert=False) client = SimpleAsyncHTTPClient(resolver=resolver, force_instance=True) rs.append((client.fetch(request), item, client)) while len(rs) > 0: if old != len(rs): print time.time(), len(rs) old = len(rs) for f in rs: if f[0].done(): rs = [_ for _ in rs if _ != f] if f[0].exception(): f[2].close() # print 'exception: ', f[0].exception() else: body = f[0].result().body if 'GoAgent' in body: print body, f[1] rss.append(f[1]) f[2].close() yield gen.sleep(0.5) if rss: with open('./all_good_ip.txt', 'a') as f: for r in rss: f.writelines(r + '\n')
from tornado.ioloop import IOLoop from tornado.netutil import Resolver, OverrideResolver from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.httpclient import HTTPRequest from tornado import gen request = HTTPRequest('http://www.baidu.com', validate_cert=False) reslover = OverrideResolver(Resolver(), mapping={"www.baidu.com": "111.13.101.208"}) client = SimpleAsyncHTTPClient(resolver=reslover) @gen.coroutine def get_response(): xxx = yield reslover.resolve('www.baidu.com', 80) print xxx rs = yield client.fetch(request) print rs.body IOLoop.current().run_sync(get_response)