def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): """The proxy_bypass function should be called with a hostname or IP without a port number or auth credentials. """ proxy_bypass = mocker.patch('requests.utils.proxy_bypass') should_bypass_proxies(url, no_proxy=None) proxy_bypass.assert_called_once_with(expected)
def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): """The proxy_bypass function should be called with a hostname or IP without a port number or auth credentials. """ proxy_bypass = mocker.patch('requests.utils.proxy_bypass') should_bypass_proxies(url, no_proxy=None) proxy_bypass.assert_called_once_with(expected)
def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') assert should_bypass_proxies(url, no_proxy=None) == expected
def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') assert should_bypass_proxies(url, no_proxy=None) == expected
def test_should_bypass_proxies_no_proxy(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not using the 'no_proxy' argument """ no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' # Test 'no_proxy' argument assert should_bypass_proxies(url, no_proxy=no_proxy) == expected
def test_should_bypass_proxies_win_registry_bad_values(monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows invalid registry settings. """ import winreg class RegHandle: def Close(self): pass ie_settings = RegHandle() def OpenKey(key, subkey): return ie_settings def QueryValueEx(key, value_name): if key is ie_settings: if value_name == "ProxyEnable": # Invalid response; Should be an int or int-y value return [""] elif value_name == "ProxyOverride": return ["192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1"] monkeypatch.setenv("http_proxy", "") monkeypatch.setenv("https_proxy", "") monkeypatch.setenv("no_proxy", "") monkeypatch.setenv("NO_PROXY", "") monkeypatch.setattr(winreg, "OpenKey", OpenKey) monkeypatch.setattr(winreg, "QueryValueEx", QueryValueEx) assert should_bypass_proxies("http://172.16.1.1/", None) is False
def test_should_bypass_proxies_no_proxy( url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not using the 'no_proxy' argument """ no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' # Test 'no_proxy' argument assert should_bypass_proxies(url, no_proxy=no_proxy) == expected
def __init__(self, configuration, pools_size=4, maxsize=None): # urllib3.PoolManager will pass all kw parameters to connectionpool # https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/poolmanager.py#L75 # noqa: E501 # https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/connectionpool.py#L680 # noqa: E501 # maxsize is the number of requests to host that are allowed in parallel # noqa: E501 # Custom SSL certificates and client certificates: http://urllib3.readthedocs.io/en/latest/advanced-usage.html # noqa: E501 # cert_reqs if configuration.verify_ssl: cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE # ca_certs if configuration.ssl_ca_cert: ca_certs = configuration.ssl_ca_cert else: # if not set certificate file, use Mozilla's root certificates. ca_certs = certifi.where() addition_pool_args = {} if configuration.assert_hostname is not None: addition_pool_args[ 'assert_hostname'] = configuration.assert_hostname # noqa: E501 if configuration.retries is not None: addition_pool_args['retries'] = configuration.retries if maxsize is None: if configuration.connection_pool_maxsize is not None: maxsize = configuration.connection_pool_maxsize else: maxsize = 4 # https pool manager if configuration.proxy and not should_bypass_proxies( configuration.host, no_proxy=configuration.no_proxy or ''): self.pool_manager = urllib3.ProxyManager( num_pools=pools_size, maxsize=maxsize, cert_reqs=cert_reqs, ca_certs=ca_certs, cert_file=configuration.cert_file, key_file=configuration.key_file, proxy_url=configuration.proxy, proxy_headers=configuration.proxy_headers, **addition_pool_args) else: self.pool_manager = urllib3.PoolManager( num_pools=pools_size, maxsize=maxsize, cert_reqs=cert_reqs, ca_certs=ca_certs, cert_file=configuration.cert_file, key_file=configuration.key_file, **addition_pool_args)
def modify_send_kwargs_for_request(self, request, send_kwargs): # This approach does not cover some (hopefully rare) cases: # * If the mount point url is in the no_proxy list the proxy config # will have already been removed. # * If the mount point url and backend url match different proxy # authorisation the one for the mount point url will be used. # However the code to do this the right way is on the session and not # easily reproducable here. proxies = send_kwargs.get('proxies') if proxies: if request.url != request._original_url: if should_bypass_proxies(request.url, proxies.get('no')): send_kwargs = send_kwargs.copy() send_kwargs['proxies'] = {} return send_kwargs
def rebuild_proxies(self, prepared_request, proxies): """This method re-evaluates the proxy configuration by considering the environment variables. If we are redirected to a URL covered by NO_PROXY, we strip the proxy configuration. Otherwise, we set missing proxy keys for this URL (in case they were stripped by a previous redirect). This method also replaces the Proxy-Authorization header where necessary. :rtype: dict """ proxies = proxies if proxies is not None else {} headers = prepared_request.headers url = prepared_request.url scheme = urlparse(url).scheme new_proxies = proxies.copy() no_proxy = proxies.get('no_proxy') bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) # if self.trust_env and not bypass_proxy: # environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) # # proxy = environ_proxies.get(scheme, environ_proxies.get('all')) # # if proxy: # new_proxies.setdefault(scheme, proxy) if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] try: username, password = get_auth_from_url(new_proxies[scheme]) except KeyError: username, password = None, None if username and password: headers['Proxy-Authorization'] = _basic_auth_str( username, password) return new_proxies
def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows registry settings """ if override is None: override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1' if compat.is_py3: import winreg else: import _winreg as winreg class RegHandle: def Close(self): pass ie_settings = RegHandle() proxyEnableValues = deque([1, "1"]) def OpenKey(key, subkey): return ie_settings def QueryValueEx(key, value_name): if key is ie_settings: if value_name == 'ProxyEnable': # this could be a string (REG_SZ) or a 32-bit number (REG_DWORD) proxyEnableValues.rotate() return [proxyEnableValues[0]] elif value_name == 'ProxyOverride': return [override] monkeypatch.setenv('http_proxy', '') monkeypatch.setenv('https_proxy', '') monkeypatch.setenv('ftp_proxy', '') monkeypatch.setenv('no_proxy', '') monkeypatch.setenv('NO_PROXY', '') monkeypatch.setattr(winreg, 'OpenKey', OpenKey) monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) assert should_bypass_proxies(url, None) == expected
def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows registry settings """ if override is None: override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1' if compat.is_py3: import winreg else: import _winreg as winreg class RegHandle: def Close(self): pass ie_settings = RegHandle() proxyEnableValues = deque([1, "1"]) def OpenKey(key, subkey): return ie_settings def QueryValueEx(key, value_name): if key is ie_settings: if value_name == 'ProxyEnable': # this could be a string (REG_SZ) or a 32-bit number (REG_DWORD) proxyEnableValues.rotate() return [proxyEnableValues[0]] elif value_name == 'ProxyOverride': return [override] monkeypatch.setenv('http_proxy', '') monkeypatch.setenv('https_proxy', '') monkeypatch.setenv('ftp_proxy', '') monkeypatch.setenv('no_proxy', '') monkeypatch.setenv('NO_PROXY', '') monkeypatch.setattr(winreg, 'OpenKey', OpenKey) monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) assert should_bypass_proxies(url, None) == expected
def _fetch_and_compile_index(index_urls, logger=None, proxy_config=None): """ Go through the index list and compile results into a single object. """ status = [] index = {} proxies_dict = {} verify = True if proxy_config: https_proxy = proxy_config.get("https_proxy", None) http_proxy = proxy_config.get("http_proxy", None) no_proxy = proxy_config.get("no_proxy", None) ca_bundle_path = proxy_config.get("proxy_ca_bundle_path", None) if https_proxy: proxies_dict["https"] = https_proxy verify = ca_bundle_path or True if http_proxy: proxies_dict["http"] = http_proxy if no_proxy: proxies_dict["no"] = no_proxy for index_url in index_urls: # TODO: # Bug in requests doesn't bypass proxies, so we do it ourselves # If this issue ever gets fixed then we can remove it # https://github.com/psf/requests/issues/4871 bypass_proxy = should_bypass_proxies(index_url, proxies_dict.get("no")) index_status = { "url": index_url, "packs": 0, "message": None, "error": None, } index_json = None try: request = requests.get( index_url, proxies=proxies_dict if not bypass_proxy else None, verify=verify if not bypass_proxy else True, ) request.raise_for_status() index_json = request.json() except ValueError as e: index_status["error"] = "malformed" index_status["message"] = repr(e) except requests.exceptions.RequestException as e: index_status["error"] = "unresponsive" index_status["message"] = repr(e) except Exception as e: index_status["error"] = "other errors" index_status["message"] = repr(e) if index_json == {}: index_status["error"] = "empty" index_status["message"] = "The index URL returned an empty object." elif type(index_json) is list: index_status["error"] = "malformed" index_status[ "message"] = "Expected an index object, got a list instead." elif index_json and "packs" not in index_json: index_status["error"] = "malformed" index_status[ "message"] = 'Index object is missing "packs" attribute.' if index_status["error"]: logger.error("Index parsing error: %s" % json_encode(index_status, indent=4)) else: # TODO: Notify on a duplicate pack aka pack being overwritten from a different index packs_data = index_json["packs"] index_status["message"] = "Success." index_status["packs"] = len(packs_data) index.update(packs_data) status.append(index_status) return index, status