def __init__(self,
proxy_url,
num_pools=10,
headers=None,
proxy_headers=None,
**connection_pool_kw):
if isinstance(proxy_url, HTTPConnectionPool):
proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host,
proxy_url.port)
proxy = parse_url(proxy_url)
if not proxy.port:
port = port_by_scheme.get(proxy.scheme, 80)
proxy = proxy._replace(port=port)
if proxy.scheme not in ("http", "https"):
raise ProxySchemeUnknown(proxy.scheme)
self.proxy = proxy
self.proxy_headers = proxy_headers or {}
connection_pool_kw['_proxy'] = self.proxy
connection_pool_kw['_proxy_headers'] = self.proxy_headers
super(ProxyManager, self).__init__(num_pools, headers,
**connection_pool_kw)
def urlopen(self, method, url, redirect=True, **kw):
"""
Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic and only sends the request-uri
portion of the ``url``.
The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
#===============================================================================================================
# add by mz
error_type = kw.get('error_type')
if error_type:
from urllib3.exceptions import LocationValueError, HostChangedError, LocationParseError, ConnectTimeoutError
from urllib3.exceptions import ProxyError, TimeoutError, ReadTimeoutError, ProtocolError, DecodeError
from urllib3.exceptions import ResponseError, ResponseNotChunked, SSLError, HTTPError, HTTPWarning, PoolError
from urllib3.exceptions import RequestError, MaxRetryError, TimeoutStateError, NewConnectionError
from urllib3.exceptions import EmptyPoolError, ClosedPoolError, SecurityWarning, SubjectAltNameWarning
from urllib3.exceptions import InsecureRequestWarning, SystemTimeWarning, InsecurePlatformWarning
from urllib3.exceptions import SNIMissingWarning, DependencyWarning, ProxySchemeUnknown, HeaderParsingError
get_error = {
"LocationValueError":
LocationValueError(),
"HostChangedError":
HostChangedError(pool=1, url=2),
"LocationParseError":
LocationParseError(url),
"ConnectTimeoutError":
ConnectTimeoutError(),
"ProxyError":
ProxyError(),
"TimeoutError":
TimeoutError(),
"ReadTimeoutError":
ReadTimeoutError(pool=1, url=2, message="ReadTimeoutError"),
"ProtocolError":
ProtocolError(),
"DecodeError":
DecodeError(),
"ResponseError":
ResponseError(),
"ResponseNotChunked":
ResponseNotChunked(),
"SSLError":
SSLError(),
"HTTPError":
HTTPError(),
"HTTPWarning":
HTTPWarning(),
"PoolError":
PoolError(pool=1, message=2),
"RequestError":
RequestError(pool=1, url=2, message="RequestError"),
"MaxRetryError":
MaxRetryError(pool=1, url=2, reason=None),
"TimeoutStateError":
TimeoutStateError(),
"NewConnectionError":
NewConnectionError(pool=1, message="NewConnectionError"),
"EmptyPoolError":
EmptyPoolError(pool=1, message="EmptyPoolError"),
"ClosedPoolError":
ClosedPoolError(pool=1, message="ClosedPoolError"),
"SecurityWarning":
SecurityWarning(),
"SubjectAltNameWarning":
SubjectAltNameWarning(),
"InsecureRequestWarning":
InsecureRequestWarning(),
"SystemTimeWarning":
SystemTimeWarning(),
"InsecurePlatformWarning":
InsecurePlatformWarning(),
"SNIMissingWarning":
SNIMissingWarning(),
"DependencyWarning":
DependencyWarning(),
"ProxySchemeUnknown":
ProxySchemeUnknown(scheme=1),
"HeaderParsingError":
HeaderParsingError(defects=1, unparsed_data=2)
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
u = parse_url(url)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw['assert_same_host'] = False
kw['redirect'] = False
if 'headers' not in kw:
kw['headers'] = self.headers
if self.proxy is not None and u.scheme == "http":
response = conn.urlopen(method, url, **kw)
else:
response = conn.urlopen(method, u.request_uri, **kw)
redirect_location = redirect and response.get_redirect_location()
if not redirect_location:
return response
# Support relative URLs for redirecting.
redirect_location = urljoin(url, redirect_location)
# RFC 2616, Section 10.3.4
if response.status == 303:
method = 'GET'
log.info("Redirecting %s -> %s" % (url, redirect_location))
kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
kw['redirect'] = redirect
return self.urlopen(method, redirect_location, **kw)