def get_proxy_key(url, dns_cache):
urlparser = urlparse(url)
ssl = urlparser.scheme == 'https'
host = urlparser.netloc.split(':')[0]
port = urlparser.port
if is_ip(host):
if not port:
raise ProxyNotPort('proxy: {} has not port'.format(url))
key = (host, port, ssl)
else:
if not port:
port = 443 if ssl else 80
ip, port = yield from dns_cache.get(host, port)
key = (ip, port, ssl)
return key
def generate_direct_connect(self,
host,
port,
ssl,
dns_cache,
recycle=True):
key = None
if is_ip(host):
ip = host.split(':')[0]
key = (ip, port, ssl)
if not key and not ssl:
ip, port = yield from dns_cache.get(host, port)
key = (ip, port, ssl)
if not key and ssl:
key = (host, port, ssl)
conn = yield from self.get_connection(key, recycle=recycle)
return conn
def get(self, host, port, uncache=False):
if is_ip(host):
return host, port
key = (host, port)
if uncache:
ipaddrs = yield from self.get_ipaddrs(host, port)
ipaddr = self.add_host(key, ipaddrs)
else:
ipaddr = self.__hosts.get(key)
if not ipaddr:
ipaddrs = yield from self.get_ipaddrs(host, port)
ipaddr = self.add_host(key, ipaddrs)
self.limit_cache()
assert ipaddr, NotFindIP(str(key))
family, type, proto, canonname, (ip, port, *_) = ipaddr
return ip, port
def _request(self,
method,
url,
params=None,
headers=None,
data=None,
cookies=None,
proxy=None,
allow_redirects=True,
recycle=None,
encoding=None):
logging.debug('[Session.request]: '
'method: {}, '
'url: {}, '
'params: {}, '
'headers: {}, '
'data: {}, '
'cookies: {}, '
'proxy: {}'.format(method, url, params, headers, data,
cookies, proxy))
encoding = encoding or self.encoding
if recycle is None:
recycle = self.recycle
if cookies:
self.cookies.update(cookies)
request = Request(method,
url,
params=params,
headers=headers,
data=data,
proxy=proxy,
cookies=self.cookies,
encoding=encoding)
host = request.url_parse_result.netloc
ssl = request.url_parse_result.scheme.lower() == 'https'
port = request.url_parse_result.port
if not port:
port = 443 if ssl else 80
# handle connection
key = None
if proxy and not ssl:
key = yield from get_proxy_key(proxy, self.dns_cache)
if not key and proxy and ssl:
_key = yield from get_proxy_key(proxy, self.dns_cache)
key = (_key[0], _key[1], host)
if not key and is_ip(host):
ip = host.split(':')[0]
key = (ip, port, ssl)
if not key and not ssl:
ip, port = yield from self.dns_cache.get(host, port)
key = (ip, port, ssl)
if not key and ssl:
key = (host, port, ssl)
conn = yield from self.adapter.get_connection(key, recycle=recycle)
# send request
yield from self.adapter.send_request(conn, request)
response = yield from self.adapter.get_response(method,
conn,
encoding=encoding)
# update cookies
self.cookies.update(response.cookies)
response.cookies = self.cookies
if method.lower() != 'connect':
self.connection_pool.recycle_connection(conn)
return response