def proxy_url_for(self, url):
"""Retrirves the corresponding proxy url for a given url. """
parsed_url = urlparse(url)
proxy = self._proxies.get(parsed_url.scheme)
if proxy:
proxy = self._fix_proxy_url(proxy)
return proxy
def _path_url(self, url):
parsed_url = urlparse(url)
path = parsed_url.path
if not path:
path = '/'
if parsed_url.query:
path = path + '?' + parsed_url.query
return path
def _prepare_url(self, original):
url = original.url
if original.params:
url_parts = urlparse(url)
delim = '&' if url_parts.query else '?'
params = urlencode(list(original.params.items()), doseq=True)
url = delim.join((url, params))
return url
def mask_proxy_url(proxy_url):
"""
Mask proxy url credentials.
:type proxy_url: str
:param proxy_url: The proxy url, i.e. https://username:[email protected]
:return: Masked proxy url, i.e. https://***:***@proxy.com
"""
mask = '*' * 3
parsed_url = urlparse(proxy_url)
if parsed_url.username:
proxy_url = proxy_url.replace(parsed_url.username, mask, 1)
if parsed_url.password:
proxy_url = proxy_url.replace(parsed_url.password, mask, 1)
return proxy_url
def _get_request_target(self, url, proxy_url):
has_proxy = proxy_url is not None
if not has_proxy:
return self._path_url(url)
# HTTP proxies expect the request_target to be the absolute url to know
# which host to establish a connection to. urllib3 also supports
# forwarding for HTTPS through the 'use_forwarding_for_https' parameter.
proxy_scheme = urlparse(proxy_url).scheme
using_https_forwarding_proxy = (proxy_scheme == 'https'
and self._proxies_kwargs.get(
'use_forwarding_for_https', False))
if using_https_forwarding_proxy or url.startswith('http:'):
return url
else:
return self._path_url(url)
def should_bypass_proxies(url):
"""
Returns whether we should bypass proxies or not.
"""
# NOTE: requests allowed for ip/cidr entries in no_proxy env that we don't
# support current as urllib only checks DNS suffix
# If the system proxy settings indicate that this URL should be bypassed,
# don't proxy.
# The proxy_bypass function is incredibly buggy on OS X in early versions
# of Python 2.6, so allow this call to fail. Only catch the specific
# exceptions we've seen, though: this call failing in other ways can reveal
# legitimate problems.
try:
if proxy_bypass(urlparse(url).netloc):
return True
except (TypeError, socket.gaierror):
pass
return False
def _urlparse(url):
if isinstance(url, six.binary_type):
# Not really necessary, but it helps to reduce noise on Python 2.x
url = url.decode('utf8')
return urlparse(url)
def _get_auth_from_url(self, url):
parsed_url = urlparse(url)
try:
return unquote(parsed_url.username), unquote(parsed_url.password)
except (AttributeError, TypeError):
return None, None
def send(self, request):
try:
proxy_url = self._proxy_config.proxy_url_for(request.url)
manager = self._get_connection_manager(request.url, proxy_url)
conn = manager.connection_from_url(request.url)
self._setup_ssl_cert(conn, request.url, self._verify)
if ensure_boolean(
os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER',
'')):
# This is currently an "experimental" feature which provides
# no guarantees of backwards compatibility. It may be subject
# to change or removal in any patch version. Anyone opting in
# to this feature should strictly pin ibm_botocore.
host = urlparse(request.url).hostname
conn.proxy_headers['host'] = host
request_target = self._get_request_target(request.url, proxy_url)
urllib_response = conn.urlopen(
method=request.method,
url=request_target,
body=request.body,
headers=request.headers,
retries=Retry(False),
assert_same_host=False,
preload_content=False,
decode_content=False,
chunked=self._chunked(request.headers),
)
http_response = ibm_botocore.awsrequest.AWSResponse(
request.url,
urllib_response.status,
urllib_response.headers,
urllib_response,
)
if not request.stream_output:
# Cause the raw stream to be exhausted immediately. We do it
# this way instead of using preload_content because
# preload_content will never buffer chunked responses
http_response.content
return http_response
except URLLib3SSLError as e:
raise SSLError(endpoint_url=request.url, error=e)
except (NewConnectionError, socket.gaierror) as e:
raise EndpointConnectionError(endpoint_url=request.url, error=e)
except ProxyError as e:
raise ProxyConnectionError(proxy_url=mask_proxy_url(proxy_url),
error=e)
except URLLib3ConnectTimeoutError as e:
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
except URLLib3ReadTimeoutError as e:
raise ReadTimeoutError(endpoint_url=request.url, error=e)
except ProtocolError as e:
raise ConnectionClosedError(error=e,
request=request,
endpoint_url=request.url)
except Exception as e:
message = 'Exception received when sending urllib3 HTTP request'
logger.debug(message, exc_info=True)
raise HTTPClientError(error=e)
def _serialize_url(self, url, event_dict, **kwargs):
event_dict['Fqdn'] = urlparse(url).netloc
