def proxy_url_for(self, url): """Retrirves the corresponding proxy url for a given url. """ parsed_url = urlparse(url) proxy = self._proxies.get(parsed_url.scheme) if proxy: proxy = self._fix_proxy_url(proxy) return proxy
def _path_url(self, url): parsed_url = urlparse(url) path = parsed_url.path if not path: path = '/' if parsed_url.query: path = path + '?' + parsed_url.query return path
def _prepare_url(self, original): url = original.url if original.params: url_parts = urlparse(url) delim = '&' if url_parts.query else '?' params = urlencode(list(original.params.items()), doseq=True) url = delim.join((url, params)) return url
def mask_proxy_url(proxy_url): """ Mask proxy url credentials. :type proxy_url: str :param proxy_url: The proxy url, i.e. https://username:[email protected] :return: Masked proxy url, i.e. https://***:***@proxy.com """ mask = '*' * 3 parsed_url = urlparse(proxy_url) if parsed_url.username: proxy_url = proxy_url.replace(parsed_url.username, mask, 1) if parsed_url.password: proxy_url = proxy_url.replace(parsed_url.password, mask, 1) return proxy_url
def _get_request_target(self, url, proxy_url): has_proxy = proxy_url is not None if not has_proxy: return self._path_url(url) # HTTP proxies expect the request_target to be the absolute url to know # which host to establish a connection to. urllib3 also supports # forwarding for HTTPS through the 'use_forwarding_for_https' parameter. proxy_scheme = urlparse(proxy_url).scheme using_https_forwarding_proxy = (proxy_scheme == 'https' and self._proxies_kwargs.get( 'use_forwarding_for_https', False)) if using_https_forwarding_proxy or url.startswith('http:'): return url else: return self._path_url(url)
def should_bypass_proxies(url): """ Returns whether we should bypass proxies or not. """ # NOTE: requests allowed for ip/cidr entries in no_proxy env that we don't # support current as urllib only checks DNS suffix # If the system proxy settings indicate that this URL should be bypassed, # don't proxy. # The proxy_bypass function is incredibly buggy on OS X in early versions # of Python 2.6, so allow this call to fail. Only catch the specific # exceptions we've seen, though: this call failing in other ways can reveal # legitimate problems. try: if proxy_bypass(urlparse(url).netloc): return True except (TypeError, socket.gaierror): pass return False
def _urlparse(url): if isinstance(url, six.binary_type): # Not really necessary, but it helps to reduce noise on Python 2.x url = url.decode('utf8') return urlparse(url)
def _get_auth_from_url(self, url): parsed_url = urlparse(url) try: return unquote(parsed_url.username), unquote(parsed_url.password) except (AttributeError, TypeError): return None, None
def send(self, request): try: proxy_url = self._proxy_config.proxy_url_for(request.url) manager = self._get_connection_manager(request.url, proxy_url) conn = manager.connection_from_url(request.url) self._setup_ssl_cert(conn, request.url, self._verify) if ensure_boolean( os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')): # This is currently an "experimental" feature which provides # no guarantees of backwards compatibility. It may be subject # to change or removal in any patch version. Anyone opting in # to this feature should strictly pin ibm_botocore. host = urlparse(request.url).hostname conn.proxy_headers['host'] = host request_target = self._get_request_target(request.url, proxy_url) urllib_response = conn.urlopen( method=request.method, url=request_target, body=request.body, headers=request.headers, retries=Retry(False), assert_same_host=False, preload_content=False, decode_content=False, chunked=self._chunked(request.headers), ) http_response = ibm_botocore.awsrequest.AWSResponse( request.url, urllib_response.status, urllib_response.headers, urllib_response, ) if not request.stream_output: # Cause the raw stream to be exhausted immediately. We do it # this way instead of using preload_content because # preload_content will never buffer chunked responses http_response.content return http_response except URLLib3SSLError as e: raise SSLError(endpoint_url=request.url, error=e) except (NewConnectionError, socket.gaierror) as e: raise EndpointConnectionError(endpoint_url=request.url, error=e) except ProxyError as e: raise ProxyConnectionError(proxy_url=mask_proxy_url(proxy_url), error=e) except URLLib3ConnectTimeoutError as e: raise ConnectTimeoutError(endpoint_url=request.url, error=e) except URLLib3ReadTimeoutError as e: raise ReadTimeoutError(endpoint_url=request.url, error=e) except ProtocolError as e: raise ConnectionClosedError(error=e, request=request, endpoint_url=request.url) except Exception as e: message = 'Exception received when sending urllib3 HTTP request' logger.debug(message, exc_info=True) raise HTTPClientError(error=e)
def _serialize_url(self, url, event_dict, **kwargs): event_dict['Fqdn'] = urlparse(url).netloc