class TestProxyConfiguration(unittest.TestCase):
def setUp(self):
self.url = 'http://localhost/'
self.auth_url = 'http://*****:*****@localhost/'
self.proxy_config = ProxyConfiguration(
proxies={'http': 'http://localhost:8081/'})
def update_http_proxy(self, url):
self.proxy_config = ProxyConfiguration(proxies={'http': url})
def test_construct_proxy_headers_with_auth(self):
headers = self.proxy_config.proxy_headers_for(self.auth_url)
proxy_auth = headers.get('Proxy-Authorization')
self.assertEqual('Basic dXNlcjpwYXNz', proxy_auth)
def test_construct_proxy_headers_without_auth(self):
headers = self.proxy_config.proxy_headers_for(self.url)
self.assertEqual({}, headers)
def test_proxy_for_url_no_slashes(self):
self.update_http_proxy('localhost:8081/')
proxy_url = self.proxy_config.proxy_url_for(self.url)
self.assertEqual('http://localhost:8081/', proxy_url)
def test_proxy_for_url_no_protocol(self):
self.update_http_proxy('//localhost:8081/')
proxy_url = self.proxy_config.proxy_url_for(self.url)
self.assertEqual('http://localhost:8081/', proxy_url)
def test_fix_proxy_url_has_protocol_http(self):
proxy_url = self.proxy_config.proxy_url_for(self.url)
self.assertEqual('http://localhost:8081/', proxy_url)
class TestProxyConfiguration(unittest.TestCase):
def setUp(self):
self.url = 'http://localhost/'
self.auth_url = 'http://*****:*****@localhost/'
self.proxy_config = ProxyConfiguration(
proxies={'http': 'http://localhost:8081/'}
)
def update_http_proxy(self, url):
self.proxy_config = ProxyConfiguration(
proxies={'http': url}
)
def test_construct_proxy_headers_with_auth(self):
headers = self.proxy_config.proxy_headers_for(self.auth_url)
proxy_auth = headers.get('Proxy-Authorization')
self.assertEqual('Basic dXNlcjpwYXNz', proxy_auth)
def test_construct_proxy_headers_without_auth(self):
headers = self.proxy_config.proxy_headers_for(self.url)
self.assertEqual({}, headers)
def test_proxy_for_url_no_slashes(self):
self.update_http_proxy('localhost:8081/')
proxy_url = self.proxy_config.proxy_url_for(self.url)
self.assertEqual('http://localhost:8081/', proxy_url)
def test_proxy_for_url_no_protocol(self):
self.update_http_proxy('//localhost:8081/')
proxy_url = self.proxy_config.proxy_url_for(self.url)
self.assertEqual('http://localhost:8081/', proxy_url)
def test_fix_proxy_url_has_protocol_http(self):
proxy_url = self.proxy_config.proxy_url_for(self.url)
self.assertEqual('http://localhost:8081/', proxy_url)
def __init__(
self,
verify: bool = True,
proxies: Dict[str, str] = None, # {scheme: url}
timeout: float = None,
max_pool_connections: int = MAX_POOL_CONNECTIONS,
socket_options=None,
client_cert=None,
proxies_config=None,
connector_args=None
):
# TODO: handle socket_options
self._verify = verify
self._proxy_config = ProxyConfiguration(proxies=proxies,
proxies_settings=proxies_config)
if isinstance(timeout, (list, tuple)):
conn_timeout, read_timeout = timeout
else:
conn_timeout = read_timeout = timeout
timeout = aiohttp.ClientTimeout(
sock_connect=conn_timeout,
sock_read=read_timeout
)
self._cert_file = None
self._key_file = None
if isinstance(client_cert, str):
self._cert_file = client_cert
elif isinstance(client_cert, tuple):
self._cert_file, self._key_file = client_cert
self._timeout = timeout
self._connector_args = connector_args
if self._connector_args is None:
# AWS has a 20 second idle timeout:
# https://forums.aws.amazon.com/message.jspa?messageID=215367
# aiohttp default timeout is 30s so set something reasonable here
self._connector_args = dict(keepalive_timeout=12)
self._max_pool_connections = max_pool_connections
self._socket_options = socket_options
if socket_options is None:
self._socket_options = []
# aiohttp handles 100 continue so we shouldn't need AWSHTTP[S]ConnectionPool
# it also pools by host so we don't need a manager, and can pass proxy via
# request so don't need proxy manager
if proxies:
proxies_settings = self._proxy_config.settings
ssl_context = self._setup_proxy_ssl_context(proxies_settings)
# TODO: add support for
# proxies_settings.get('proxy_use_forwarding_for_https')
else:
ssl_context = self._get_ssl_context()
# inline self._setup_ssl_cert
if bool(verify):
ca_certs = get_cert_path(verify)
if ca_certs:
ssl_context.load_verify_locations(ca_certs, None, None)
self._connector = aiohttp.TCPConnector(
limit=max_pool_connections,
verify_ssl=bool(verify),
ssl=ssl_context,
**connector_args)
self._session: Optional[aiohttp.ClientSession] = None
class AIOHTTPSession:
def __init__(
self,
verify: bool = True,
proxies: Dict[str, str] = None, # {scheme: url}
timeout: float = None,
max_pool_connections: int = MAX_POOL_CONNECTIONS,
socket_options=None,
client_cert=None,
proxies_config=None,
connector_args=None
):
# TODO: handle socket_options
self._verify = verify
self._proxy_config = ProxyConfiguration(proxies=proxies,
proxies_settings=proxies_config)
if isinstance(timeout, (list, tuple)):
conn_timeout, read_timeout = timeout
else:
conn_timeout = read_timeout = timeout
timeout = aiohttp.ClientTimeout(
sock_connect=conn_timeout,
sock_read=read_timeout
)
self._cert_file = None
self._key_file = None
if isinstance(client_cert, str):
self._cert_file = client_cert
elif isinstance(client_cert, tuple):
self._cert_file, self._key_file = client_cert
self._timeout = timeout
self._connector_args = connector_args
if self._connector_args is None:
# AWS has a 20 second idle timeout:
# https://forums.aws.amazon.com/message.jspa?messageID=215367
# aiohttp default timeout is 30s so set something reasonable here
self._connector_args = dict(keepalive_timeout=12)
self._max_pool_connections = max_pool_connections
self._socket_options = socket_options
if socket_options is None:
self._socket_options = []
# aiohttp handles 100 continue so we shouldn't need AWSHTTP[S]ConnectionPool
# it also pools by host so we don't need a manager, and can pass proxy via
# request so don't need proxy manager
if proxies:
proxies_settings = self._proxy_config.settings
ssl_context = self._setup_proxy_ssl_context(proxies_settings)
# TODO: add support for
# proxies_settings.get('proxy_use_forwarding_for_https')
else:
ssl_context = self._get_ssl_context()
# inline self._setup_ssl_cert
if bool(verify):
ca_certs = get_cert_path(verify)
if ca_certs:
ssl_context.load_verify_locations(ca_certs, None, None)
self._connector = aiohttp.TCPConnector(
limit=max_pool_connections,
verify_ssl=bool(verify),
ssl=ssl_context,
**connector_args)
self._session: Optional[aiohttp.ClientSession] = None
async def __aenter__(self):
self._session = aiohttp.ClientSession(
connector=self._connector,
timeout=self._timeout,
skip_auto_headers={'CONTENT-TYPE'},
response_class=ClientResponseProxy,
auto_decompress=False)
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self._session.__aexit__(exc_type, exc_val, exc_tb)
def _get_ssl_context(self):
ssl_context = create_urllib3_context()
if self._cert_file:
ssl_context.load_cert_chain(self._cert_file, self._key_file)
return ssl_context
def _setup_proxy_ssl_context(self, proxies_settings):
proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle')
proxy_cert = proxies_settings.get('proxy_client_cert')
if proxy_ca_bundle is None and proxy_cert is None:
return None
context = self._get_ssl_context()
try:
# urllib3 disables this by default but we need
# it for proper proxy tls negotiation.
context.check_hostname = True
if proxy_ca_bundle is not None:
context.load_verify_locations(cafile=proxy_ca_bundle)
if isinstance(proxy_cert, tuple):
context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1])
elif isinstance(proxy_cert, str):
context.load_cert_chain(proxy_cert)
return context
except IOError as e:
raise InvalidProxiesConfigError(error=e)
async def send(self, request):
proxy_url = self._proxy_config.proxy_url_for(request.url)
try:
url = request.url
headers = request.headers
data = request.body
# https://github.com/boto/botocore/issues/1255
headers['Accept-Encoding'] = 'identity'
headers_ = MultiDict(
(z[0], _text(z[1], encoding='utf-8')) for z in headers.items())
if isinstance(data, io.IOBase):
data = _IOBaseWrapper(data)
url = URL(url, encoded=True)
resp = await self._session.request(
request.method, url=url, headers=headers_, data=data, proxy=proxy_url)
if not request.stream_output:
# Cause the raw stream to be exhausted immediately. We do it
# this way instead of using preload_content because
# preload_content will never buffer chunked responses
await resp.read()
return resp
except ClientSSLError as e:
raise SSLError(endpoint_url=request.url, error=e)
except (ClientConnectorError, socket.gaierror) as e:
raise EndpointConnectionError(endpoint_url=request.url, error=e)
except (ClientProxyConnectionError, ClientHttpProxyError) as e:
raise ProxyConnectionError(proxy_url=proxy_url, error=e)
except ServerTimeoutError as e:
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
except asyncio.TimeoutError as e:
raise ReadTimeoutError(endpoint_url=request.url, error=e)
except ServerDisconnectedError as e:
raise ConnectionClosedError(
error=e,
request=request,
endpoint_url=request.url
)
except Exception as e:
message = 'Exception received when sending urllib3 HTTP request'
logger.debug(message, exc_info=True)
raise HTTPClientError(error=e)
def update_http_proxy(self, url):
self.proxy_config = ProxyConfiguration(proxies={'http': url})
def setUp(self):
self.url = 'http://localhost/'
self.auth_url = 'http://*****:*****@localhost/'
self.proxy_config = ProxyConfiguration(
proxies={'http': 'http://localhost:8081/'})
def update_http_proxy(self, url):
self.proxy_config = ProxyConfiguration(
proxies={'http': url}
)
def setUp(self):
self.url = 'http://localhost/'
self.auth_url = 'http://*****:*****@localhost/'
self.proxy_config = ProxyConfiguration(
proxies={'http': 'http://localhost:8081/'}
)
