def get_proxy_key(url, dns_cache): urlparser = urlparse(url) ssl = urlparser.scheme == 'https' host = urlparser.netloc.split(':')[0] port = urlparser.port if is_ip(host): if not port: raise ProxyNotPort('proxy: {} has not port'.format(url)) key = (host, port, ssl) else: if not port: port = 443 if ssl else 80 ip, port = yield from dns_cache.get(host, port) key = (ip, port, ssl) return key
def generate_direct_connect(self, host, port, ssl, dns_cache, recycle=True): key = None if is_ip(host): ip = host.split(':')[0] key = (ip, port, ssl) if not key and not ssl: ip, port = yield from dns_cache.get(host, port) key = (ip, port, ssl) if not key and ssl: key = (host, port, ssl) conn = yield from self.get_connection(key, recycle=recycle) return conn
def get(self, host, port, uncache=False): if is_ip(host): return host, port key = (host, port) if uncache: ipaddrs = yield from self.get_ipaddrs(host, port) ipaddr = self.add_host(key, ipaddrs) else: ipaddr = self.__hosts.get(key) if not ipaddr: ipaddrs = yield from self.get_ipaddrs(host, port) ipaddr = self.add_host(key, ipaddrs) self.limit_cache() assert ipaddr, NotFindIP(str(key)) family, type, proto, canonname, (ip, port, *_) = ipaddr return ip, port
def _request(self, method, url, params=None, headers=None, data=None, cookies=None, proxy=None, allow_redirects=True, recycle=None, encoding=None): logging.debug('[Session.request]: ' 'method: {}, ' 'url: {}, ' 'params: {}, ' 'headers: {}, ' 'data: {}, ' 'cookies: {}, ' 'proxy: {}'.format(method, url, params, headers, data, cookies, proxy)) encoding = encoding or self.encoding if recycle is None: recycle = self.recycle if cookies: self.cookies.update(cookies) request = Request(method, url, params=params, headers=headers, data=data, proxy=proxy, cookies=self.cookies, encoding=encoding) host = request.url_parse_result.netloc ssl = request.url_parse_result.scheme.lower() == 'https' port = request.url_parse_result.port if not port: port = 443 if ssl else 80 # handle connection key = None if proxy and not ssl: key = yield from get_proxy_key(proxy, self.dns_cache) if not key and proxy and ssl: _key = yield from get_proxy_key(proxy, self.dns_cache) key = (_key[0], _key[1], host) if not key and is_ip(host): ip = host.split(':')[0] key = (ip, port, ssl) if not key and not ssl: ip, port = yield from self.dns_cache.get(host, port) key = (ip, port, ssl) if not key and ssl: key = (host, port, ssl) conn = yield from self.adapter.get_connection(key, recycle=recycle) # send request yield from self.adapter.send_request(conn, request) response = yield from self.adapter.get_response(method, conn, encoding=encoding) # update cookies self.cookies.update(response.cookies) response.cookies = self.cookies if method.lower() != 'connect': self.connection_pool.recycle_connection(conn) return response