class TestProxyConfiguration(unittest.TestCase): def setUp(self): self.url = 'http://localhost/' self.auth_url = 'http://*****:*****@localhost/' self.proxy_config = ProxyConfiguration( proxies={'http': 'http://localhost:8081/'}) def update_http_proxy(self, url): self.proxy_config = ProxyConfiguration(proxies={'http': url}) def test_construct_proxy_headers_with_auth(self): headers = self.proxy_config.proxy_headers_for(self.auth_url) proxy_auth = headers.get('Proxy-Authorization') self.assertEqual('Basic dXNlcjpwYXNz', proxy_auth) def test_construct_proxy_headers_without_auth(self): headers = self.proxy_config.proxy_headers_for(self.url) self.assertEqual({}, headers) def test_proxy_for_url_no_slashes(self): self.update_http_proxy('localhost:8081/') proxy_url = self.proxy_config.proxy_url_for(self.url) self.assertEqual('http://localhost:8081/', proxy_url) def test_proxy_for_url_no_protocol(self): self.update_http_proxy('//localhost:8081/') proxy_url = self.proxy_config.proxy_url_for(self.url) self.assertEqual('http://localhost:8081/', proxy_url) def test_fix_proxy_url_has_protocol_http(self): proxy_url = self.proxy_config.proxy_url_for(self.url) self.assertEqual('http://localhost:8081/', proxy_url)
class TestProxyConfiguration(unittest.TestCase): def setUp(self): self.url = 'http://localhost/' self.auth_url = 'http://*****:*****@localhost/' self.proxy_config = ProxyConfiguration( proxies={'http': 'http://localhost:8081/'} ) def update_http_proxy(self, url): self.proxy_config = ProxyConfiguration( proxies={'http': url} ) def test_construct_proxy_headers_with_auth(self): headers = self.proxy_config.proxy_headers_for(self.auth_url) proxy_auth = headers.get('Proxy-Authorization') self.assertEqual('Basic dXNlcjpwYXNz', proxy_auth) def test_construct_proxy_headers_without_auth(self): headers = self.proxy_config.proxy_headers_for(self.url) self.assertEqual({}, headers) def test_proxy_for_url_no_slashes(self): self.update_http_proxy('localhost:8081/') proxy_url = self.proxy_config.proxy_url_for(self.url) self.assertEqual('http://localhost:8081/', proxy_url) def test_proxy_for_url_no_protocol(self): self.update_http_proxy('//localhost:8081/') proxy_url = self.proxy_config.proxy_url_for(self.url) self.assertEqual('http://localhost:8081/', proxy_url) def test_fix_proxy_url_has_protocol_http(self): proxy_url = self.proxy_config.proxy_url_for(self.url) self.assertEqual('http://localhost:8081/', proxy_url)
def __init__( self, verify: bool = True, proxies: Dict[str, str] = None, # {scheme: url} timeout: float = None, max_pool_connections: int = MAX_POOL_CONNECTIONS, socket_options=None, client_cert=None, proxies_config=None, connector_args=None ): # TODO: handle socket_options self._verify = verify self._proxy_config = ProxyConfiguration(proxies=proxies, proxies_settings=proxies_config) if isinstance(timeout, (list, tuple)): conn_timeout, read_timeout = timeout else: conn_timeout = read_timeout = timeout timeout = aiohttp.ClientTimeout( sock_connect=conn_timeout, sock_read=read_timeout ) self._cert_file = None self._key_file = None if isinstance(client_cert, str): self._cert_file = client_cert elif isinstance(client_cert, tuple): self._cert_file, self._key_file = client_cert self._timeout = timeout self._connector_args = connector_args if self._connector_args is None: # AWS has a 20 second idle timeout: # https://forums.aws.amazon.com/message.jspa?messageID=215367 # aiohttp default timeout is 30s so set something reasonable here self._connector_args = dict(keepalive_timeout=12) self._max_pool_connections = max_pool_connections self._socket_options = socket_options if socket_options is None: self._socket_options = [] # aiohttp handles 100 continue so we shouldn't need AWSHTTP[S]ConnectionPool # it also pools by host so we don't need a manager, and can pass proxy via # request so don't need proxy manager if proxies: proxies_settings = self._proxy_config.settings ssl_context = self._setup_proxy_ssl_context(proxies_settings) # TODO: add support for # proxies_settings.get('proxy_use_forwarding_for_https') else: ssl_context = self._get_ssl_context() # inline self._setup_ssl_cert if bool(verify): ca_certs = get_cert_path(verify) if ca_certs: ssl_context.load_verify_locations(ca_certs, None, None) self._connector = aiohttp.TCPConnector( limit=max_pool_connections, verify_ssl=bool(verify), ssl=ssl_context, **connector_args) self._session: Optional[aiohttp.ClientSession] = None
class AIOHTTPSession: def __init__( self, verify: bool = True, proxies: Dict[str, str] = None, # {scheme: url} timeout: float = None, max_pool_connections: int = MAX_POOL_CONNECTIONS, socket_options=None, client_cert=None, proxies_config=None, connector_args=None ): # TODO: handle socket_options self._verify = verify self._proxy_config = ProxyConfiguration(proxies=proxies, proxies_settings=proxies_config) if isinstance(timeout, (list, tuple)): conn_timeout, read_timeout = timeout else: conn_timeout = read_timeout = timeout timeout = aiohttp.ClientTimeout( sock_connect=conn_timeout, sock_read=read_timeout ) self._cert_file = None self._key_file = None if isinstance(client_cert, str): self._cert_file = client_cert elif isinstance(client_cert, tuple): self._cert_file, self._key_file = client_cert self._timeout = timeout self._connector_args = connector_args if self._connector_args is None: # AWS has a 20 second idle timeout: # https://forums.aws.amazon.com/message.jspa?messageID=215367 # aiohttp default timeout is 30s so set something reasonable here self._connector_args = dict(keepalive_timeout=12) self._max_pool_connections = max_pool_connections self._socket_options = socket_options if socket_options is None: self._socket_options = [] # aiohttp handles 100 continue so we shouldn't need AWSHTTP[S]ConnectionPool # it also pools by host so we don't need a manager, and can pass proxy via # request so don't need proxy manager if proxies: proxies_settings = self._proxy_config.settings ssl_context = self._setup_proxy_ssl_context(proxies_settings) # TODO: add support for # proxies_settings.get('proxy_use_forwarding_for_https') else: ssl_context = self._get_ssl_context() # inline self._setup_ssl_cert if bool(verify): ca_certs = get_cert_path(verify) if ca_certs: ssl_context.load_verify_locations(ca_certs, None, None) self._connector = aiohttp.TCPConnector( limit=max_pool_connections, verify_ssl=bool(verify), ssl=ssl_context, **connector_args) self._session: Optional[aiohttp.ClientSession] = None async def __aenter__(self): self._session = aiohttp.ClientSession( connector=self._connector, timeout=self._timeout, skip_auto_headers={'CONTENT-TYPE'}, response_class=ClientResponseProxy, auto_decompress=False) return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self._session.__aexit__(exc_type, exc_val, exc_tb) def _get_ssl_context(self): ssl_context = create_urllib3_context() if self._cert_file: ssl_context.load_cert_chain(self._cert_file, self._key_file) return ssl_context def _setup_proxy_ssl_context(self, proxies_settings): proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle') proxy_cert = proxies_settings.get('proxy_client_cert') if proxy_ca_bundle is None and proxy_cert is None: return None context = self._get_ssl_context() try: # urllib3 disables this by default but we need # it for proper proxy tls negotiation. context.check_hostname = True if proxy_ca_bundle is not None: context.load_verify_locations(cafile=proxy_ca_bundle) if isinstance(proxy_cert, tuple): context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1]) elif isinstance(proxy_cert, str): context.load_cert_chain(proxy_cert) return context except IOError as e: raise InvalidProxiesConfigError(error=e) async def send(self, request): proxy_url = self._proxy_config.proxy_url_for(request.url) try: url = request.url headers = request.headers data = request.body # https://github.com/boto/botocore/issues/1255 headers['Accept-Encoding'] = 'identity' headers_ = MultiDict( (z[0], _text(z[1], encoding='utf-8')) for z in headers.items()) if isinstance(data, io.IOBase): data = _IOBaseWrapper(data) url = URL(url, encoded=True) resp = await self._session.request( request.method, url=url, headers=headers_, data=data, proxy=proxy_url) if not request.stream_output: # Cause the raw stream to be exhausted immediately. We do it # this way instead of using preload_content because # preload_content will never buffer chunked responses await resp.read() return resp except ClientSSLError as e: raise SSLError(endpoint_url=request.url, error=e) except (ClientConnectorError, socket.gaierror) as e: raise EndpointConnectionError(endpoint_url=request.url, error=e) except (ClientProxyConnectionError, ClientHttpProxyError) as e: raise ProxyConnectionError(proxy_url=proxy_url, error=e) except ServerTimeoutError as e: raise ConnectTimeoutError(endpoint_url=request.url, error=e) except asyncio.TimeoutError as e: raise ReadTimeoutError(endpoint_url=request.url, error=e) except ServerDisconnectedError as e: raise ConnectionClosedError( error=e, request=request, endpoint_url=request.url ) except Exception as e: message = 'Exception received when sending urllib3 HTTP request' logger.debug(message, exc_info=True) raise HTTPClientError(error=e)
def update_http_proxy(self, url): self.proxy_config = ProxyConfiguration(proxies={'http': url})
def setUp(self): self.url = 'http://localhost/' self.auth_url = 'http://*****:*****@localhost/' self.proxy_config = ProxyConfiguration( proxies={'http': 'http://localhost:8081/'})
def update_http_proxy(self, url): self.proxy_config = ProxyConfiguration( proxies={'http': url} )
def setUp(self): self.url = 'http://localhost/' self.auth_url = 'http://*****:*****@localhost/' self.proxy_config = ProxyConfiguration( proxies={'http': 'http://localhost:8081/'} )