def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker):
"""The proxy_bypass function should be called with a hostname or IP without
a port number or auth credentials.
"""
proxy_bypass = mocker.patch('requests.utils.proxy_bypass')
should_bypass_proxies(url, no_proxy=None)
proxy_bypass.assert_called_once_with(expected)
def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker):
"""The proxy_bypass function should be called with a hostname or IP without
a port number or auth credentials.
"""
proxy_bypass = mocker.patch('requests.utils.proxy_bypass')
should_bypass_proxies(url, no_proxy=None)
proxy_bypass.assert_called_once_with(expected)
def test_should_bypass_proxies(url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not
"""
monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000')
monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000')
assert should_bypass_proxies(url, no_proxy=None) == expected
def test_should_bypass_proxies(url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not
"""
monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
assert should_bypass_proxies(url, no_proxy=None) == expected
def test_should_bypass_proxies_no_proxy(url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not using the 'no_proxy' argument
"""
no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
# Test 'no_proxy' argument
assert should_bypass_proxies(url, no_proxy=no_proxy) == expected
def test_should_bypass_proxies_win_registry_bad_values(monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not with Windows invalid registry settings.
"""
import winreg
class RegHandle:
def Close(self):
pass
ie_settings = RegHandle()
def OpenKey(key, subkey):
return ie_settings
def QueryValueEx(key, value_name):
if key is ie_settings:
if value_name == "ProxyEnable":
# Invalid response; Should be an int or int-y value
return [""]
elif value_name == "ProxyOverride":
return ["192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1"]
monkeypatch.setenv("http_proxy", "")
monkeypatch.setenv("https_proxy", "")
monkeypatch.setenv("no_proxy", "")
monkeypatch.setenv("NO_PROXY", "")
monkeypatch.setattr(winreg, "OpenKey", OpenKey)
monkeypatch.setattr(winreg, "QueryValueEx", QueryValueEx)
assert should_bypass_proxies("http://172.16.1.1/", None) is False
def test_should_bypass_proxies_no_proxy(
url, expected, monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not using the 'no_proxy' argument
"""
no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1'
# Test 'no_proxy' argument
assert should_bypass_proxies(url, no_proxy=no_proxy) == expected
def __init__(self, configuration, pools_size=4, maxsize=None):
# urllib3.PoolManager will pass all kw parameters to connectionpool
# https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/poolmanager.py#L75 # noqa: E501
# https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/connectionpool.py#L680 # noqa: E501
# maxsize is the number of requests to host that are allowed in parallel # noqa: E501
# Custom SSL certificates and client certificates: http://urllib3.readthedocs.io/en/latest/advanced-usage.html # noqa: E501
# cert_reqs
if configuration.verify_ssl:
cert_reqs = ssl.CERT_REQUIRED
else:
cert_reqs = ssl.CERT_NONE
# ca_certs
if configuration.ssl_ca_cert:
ca_certs = configuration.ssl_ca_cert
else:
# if not set certificate file, use Mozilla's root certificates.
ca_certs = certifi.where()
addition_pool_args = {}
if configuration.assert_hostname is not None:
addition_pool_args[
'assert_hostname'] = configuration.assert_hostname # noqa: E501
if configuration.retries is not None:
addition_pool_args['retries'] = configuration.retries
if maxsize is None:
if configuration.connection_pool_maxsize is not None:
maxsize = configuration.connection_pool_maxsize
else:
maxsize = 4
# https pool manager
if configuration.proxy and not should_bypass_proxies(
configuration.host, no_proxy=configuration.no_proxy or ''):
self.pool_manager = urllib3.ProxyManager(
num_pools=pools_size,
maxsize=maxsize,
cert_reqs=cert_reqs,
ca_certs=ca_certs,
cert_file=configuration.cert_file,
key_file=configuration.key_file,
proxy_url=configuration.proxy,
proxy_headers=configuration.proxy_headers,
**addition_pool_args)
else:
self.pool_manager = urllib3.PoolManager(
num_pools=pools_size,
maxsize=maxsize,
cert_reqs=cert_reqs,
ca_certs=ca_certs,
cert_file=configuration.cert_file,
key_file=configuration.key_file,
**addition_pool_args)
def modify_send_kwargs_for_request(self, request, send_kwargs):
# This approach does not cover some (hopefully rare) cases:
# * If the mount point url is in the no_proxy list the proxy config
# will have already been removed.
# * If the mount point url and backend url match different proxy
# authorisation the one for the mount point url will be used.
# However the code to do this the right way is on the session and not
# easily reproducable here.
proxies = send_kwargs.get('proxies')
if proxies:
if request.url != request._original_url:
if should_bypass_proxies(request.url, proxies.get('no')):
send_kwargs = send_kwargs.copy()
send_kwargs['proxies'] = {}
return send_kwargs
def rebuild_proxies(self, prepared_request, proxies):
"""This method re-evaluates the proxy configuration by considering the
environment variables. If we are redirected to a URL covered by
NO_PROXY, we strip the proxy configuration. Otherwise, we set missing
proxy keys for this URL (in case they were stripped by a previous
redirect).
This method also replaces the Proxy-Authorization header where
necessary.
:rtype: dict
"""
proxies = proxies if proxies is not None else {}
headers = prepared_request.headers
url = prepared_request.url
scheme = urlparse(url).scheme
new_proxies = proxies.copy()
no_proxy = proxies.get('no_proxy')
bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy)
# if self.trust_env and not bypass_proxy:
# environ_proxies = get_environ_proxies(url, no_proxy=no_proxy)
#
# proxy = environ_proxies.get(scheme, environ_proxies.get('all'))
#
# if proxy:
# new_proxies.setdefault(scheme, proxy)
if 'Proxy-Authorization' in headers:
del headers['Proxy-Authorization']
try:
username, password = get_auth_from_url(new_proxies[scheme])
except KeyError:
username, password = None, None
if username and password:
headers['Proxy-Authorization'] = _basic_auth_str(
username, password)
return new_proxies
def test_should_bypass_proxies_win_registry(url, expected, override,
monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not with Windows registry settings
"""
if override is None:
override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1'
if compat.is_py3:
import winreg
else:
import _winreg as winreg
class RegHandle:
def Close(self):
pass
ie_settings = RegHandle()
proxyEnableValues = deque([1, "1"])
def OpenKey(key, subkey):
return ie_settings
def QueryValueEx(key, value_name):
if key is ie_settings:
if value_name == 'ProxyEnable':
# this could be a string (REG_SZ) or a 32-bit number (REG_DWORD)
proxyEnableValues.rotate()
return [proxyEnableValues[0]]
elif value_name == 'ProxyOverride':
return [override]
monkeypatch.setenv('http_proxy', '')
monkeypatch.setenv('https_proxy', '')
monkeypatch.setenv('ftp_proxy', '')
monkeypatch.setenv('no_proxy', '')
monkeypatch.setenv('NO_PROXY', '')
monkeypatch.setattr(winreg, 'OpenKey', OpenKey)
monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx)
assert should_bypass_proxies(url, None) == expected
def test_should_bypass_proxies_win_registry(url, expected, override,
monkeypatch):
"""Tests for function should_bypass_proxies to check if proxy
can be bypassed or not with Windows registry settings
"""
if override is None:
override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1'
if compat.is_py3:
import winreg
else:
import _winreg as winreg
class RegHandle:
def Close(self):
pass
ie_settings = RegHandle()
proxyEnableValues = deque([1, "1"])
def OpenKey(key, subkey):
return ie_settings
def QueryValueEx(key, value_name):
if key is ie_settings:
if value_name == 'ProxyEnable':
# this could be a string (REG_SZ) or a 32-bit number (REG_DWORD)
proxyEnableValues.rotate()
return [proxyEnableValues[0]]
elif value_name == 'ProxyOverride':
return [override]
monkeypatch.setenv('http_proxy', '')
monkeypatch.setenv('https_proxy', '')
monkeypatch.setenv('ftp_proxy', '')
monkeypatch.setenv('no_proxy', '')
monkeypatch.setenv('NO_PROXY', '')
monkeypatch.setattr(winreg, 'OpenKey', OpenKey)
monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx)
assert should_bypass_proxies(url, None) == expected
def _fetch_and_compile_index(index_urls, logger=None, proxy_config=None):
"""
Go through the index list and compile results into a single object.
"""
status = []
index = {}
proxies_dict = {}
verify = True
if proxy_config:
https_proxy = proxy_config.get("https_proxy", None)
http_proxy = proxy_config.get("http_proxy", None)
no_proxy = proxy_config.get("no_proxy", None)
ca_bundle_path = proxy_config.get("proxy_ca_bundle_path", None)
if https_proxy:
proxies_dict["https"] = https_proxy
verify = ca_bundle_path or True
if http_proxy:
proxies_dict["http"] = http_proxy
if no_proxy:
proxies_dict["no"] = no_proxy
for index_url in index_urls:
# TODO:
# Bug in requests doesn't bypass proxies, so we do it ourselves
# If this issue ever gets fixed then we can remove it
# https://github.com/psf/requests/issues/4871
bypass_proxy = should_bypass_proxies(index_url, proxies_dict.get("no"))
index_status = {
"url": index_url,
"packs": 0,
"message": None,
"error": None,
}
index_json = None
try:
request = requests.get(
index_url,
proxies=proxies_dict if not bypass_proxy else None,
verify=verify if not bypass_proxy else True,
)
request.raise_for_status()
index_json = request.json()
except ValueError as e:
index_status["error"] = "malformed"
index_status["message"] = repr(e)
except requests.exceptions.RequestException as e:
index_status["error"] = "unresponsive"
index_status["message"] = repr(e)
except Exception as e:
index_status["error"] = "other errors"
index_status["message"] = repr(e)
if index_json == {}:
index_status["error"] = "empty"
index_status["message"] = "The index URL returned an empty object."
elif type(index_json) is list:
index_status["error"] = "malformed"
index_status[
"message"] = "Expected an index object, got a list instead."
elif index_json and "packs" not in index_json:
index_status["error"] = "malformed"
index_status[
"message"] = 'Index object is missing "packs" attribute.'
if index_status["error"]:
logger.error("Index parsing error: %s" %
json_encode(index_status, indent=4))
else:
# TODO: Notify on a duplicate pack aka pack being overwritten from a different index
packs_data = index_json["packs"]
index_status["message"] = "Success."
index_status["packs"] = len(packs_data)
index.update(packs_data)
status.append(index_status)
return index, status
