def test_basic_https_proxy_request(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
self.assert_request_sent()
def test_aws_connection_classes_are_used(self):
session = URLLib3Session()
# ensure the pool manager is using the correct classes
http_class = self.pool_manager.pool_classes_by_scheme.get('http')
self.assertIs(http_class, AWSHTTPConnectionPool)
https_class = self.pool_manager.pool_classes_by_scheme.get('https')
self.assertIs(https_class, AWSHTTPSConnectionPool)
def test_catches_proxy_error(self):
self.connection.urlopen.side_effect = ProxyError('test', None)
session = URLLib3Session(proxies={'http': 'http://*****:*****@proxy.com'})
with pytest.raises(ProxyConnectionError) as e:
session.send(self.request.prepare())
assert 'user:pass' not in str(e.value)
assert 'http://***:***@proxy.com' in str(e.value)
def test_patch_http_requests_send(mock_context_wrapper):
if sys.version_info[0] != 3:
pytest.skip("this test requires python 3+, skipping")
patch_http_requests(mock_context_wrapper, None, None)
assert hasattr(BotocoreSession.send, "__wrapped__")
assert hasattr(BotocoreVendoredSession.send, "__wrapped__")
botocore_session = BotocoreSession()
botocore_session.send(AWSRequest(method="GET", url="https://www.iopipe.com"))
botocore_vendored_session = BotocoreVendoredSession()
request = PreparedRequest()
request.prepare(method="GET", url="https://www.iopipe.com")
botocore_vendored_session.send(request)
def test_proxies_config_settings_unknown_config(self):
proxies = {'http': 'http://proxy.com'}
proxies_config = {
'proxy_ca_bundle': None,
'proxy_client_cert': None,
'proxy_use_forwarding_for_https': True,
'proxy_not_a_real_arg': 'do not pass'
}
use_forwarding = proxies_config['proxy_use_forwarding_for_https']
session = URLLib3Session(
proxies=proxies,
proxies_config=proxies_config
)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
use_forwarding_for_https=use_forwarding
)
self.assertNotIn(
'proxy_not_a_real_arg',
self.proxy_manager_fun.call_args
)
self.assert_request_sent(url=self.request.url)
def test_forwards_max_pool_size(self):
URLLib3Session(max_pool_connections=22)
self.pool_manager_cls.assert_called_with(
maxsize=22,
timeout=ANY,
strict=True,
ssl_context=ANY,
)
def test_chunked_encoding_is_set_with_header(self):
session = URLLib3Session()
self.request.headers['Transfer-Encoding'] = 'chunked'
session.send(self.request.prepare())
self.assert_request_sent(
chunked=True,
headers={'Transfer-Encoding': 'chunked'},
)
def send_to_es(self, path, method="GET", payload={}):
"""Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
Args:
path (str): path to send to ES
method (str, optional): HTTP method default:GET
payload (dict, optional): additional payload used during POST or PUT
Returns:
dict: json answer converted in dict
Raises:
#: Error during ES communication
ES_Exception: Description
"""
if not path.startswith("/"):
path = "/" + path
es_region = self.cfg["es_endpoint"].split(".")[1]
headers = {
"Host": self.cfg["es_endpoint"],
"Content-Type": "application/json"
}
# send to ES with exponential backoff
retries = 0
while retries < int(self.cfg["es_max_retry"]):
if retries > 0:
seconds = (2**retries) * .1
time.sleep(seconds)
req = AWSRequest(method=method,
url="https://{}{}".format(self.cfg["es_endpoint"],
quote(path)),
data=json.dumps(payload),
params={"format": "json"},
headers=headers)
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
SigV4Auth(credentials, 'es', es_region).add_auth(req)
try:
preq = req.prepare()
session = URLLib3Session()
res = session.send(preq)
if res.status_code >= 200 and res.status_code <= 299:
return json.loads(res.content)
else:
raise ES_Exception(res.status_code, res._content)
except ES_Exception as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def test_https_proxy_scheme_with_http_url(self):
proxies = {'http': 'https://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
)
self.assert_request_sent(url=self.request.url)
def test_basic_proxy_request_caches_manager(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
# assert we created the proxy manager
self.assert_proxy_manager_call(proxies['https'], proxy_headers={})
session.send(self.request.prepare())
# assert that we did not create another proxy manager
self.assertEqual(self.proxy_manager_fun.call_count, 1)
def get_uri(prefix, uri):
try:
session = URLLib3Session()
r = session.send(AWSRequest('GET', uri).prepare())
if r.status_code == 200:
return r.text
else:
raise ResourceLoadingError("received non 200 status code of %s" %
(r.status_code))
except Exception as e:
raise ResourceLoadingError('Unable to retrieve %s: %s' % (uri, e))
def post_data_to_es(payload, region, creds, host, path, method='POST', proto='https://'):
'''Post data to ES endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method, url=proto + host + urllib.quote(path), data=payload, headers={'Host': host, 'Content-Type' : 'application/json'})
SigV4Auth(creds, 'es', region).add_auth(req)
#http_session = BotocoreHTTPSession()
http_session = URLLib3Session()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
return res._content
else:
raise ES_Exception(res.status_code, res._content)
def test_https_proxy_scheme_forwarding_https_url(self):
proxies = {'https': 'https://proxy.com'}
proxies_config = {"proxy_use_forwarding_for_https": True}
session = URLLib3Session(proxies=proxies, proxies_config=proxies_config)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
use_forwarding_for_https=True,
)
self.assert_request_sent(url=self.request.url)
def test_basic_https_proxy_with_client_cert(self):
proxies = {'https': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies, client_cert='/some/cert')
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['https'],
proxy_headers={},
cert_file='/some/cert',
key_file=None,
)
self.assert_request_sent()
def __init__(self, host, endpoint_prefix, event_emitter,
response_parser_factory=None, http_session=None):
self._endpoint_prefix = endpoint_prefix
self._event_emitter = event_emitter
self.host = host
self._lock = threading.Lock()
if response_parser_factory is None:
response_parser_factory = parsers.ResponseParserFactory()
self._response_parser_factory = response_parser_factory
self.http_session = http_session
if self.http_session is None:
self.http_session = URLLib3Session()
def test_proxy_ssl_context_uses_check_hostname(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'https://proxy.com'}
proxies_config = {'proxy_client_cert': "path/to/cert"}
with patch('botocore.httpsession.create_urllib3_context'):
session = URLLib3Session(proxies=proxies,
client_cert=cert,
proxies_config=proxies_config)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
last_call = self.proxy_manager_fun.call_args[-1]
self.assertIs(last_call['ssl_context'].check_hostname, True)
def test_basic_http_proxy_request(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
self.proxy_manager_fun.assert_any_call(
proxies['http'],
proxy_headers={},
maxsize=ANY,
timeout=ANY,
strict=True,
ssl_context=ANY,
)
self.assert_request_sent(url=self.request.url)
def test_session_forwards_socket_options_to_proxy_manager(self):
proxies = {'http': 'http://proxy.com'}
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
session = URLLib3Session(
proxies=proxies,
socket_options=socket_options,
)
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
socket_options=socket_options,
)
def test_urllib3_proxies_kwargs_included(self):
cert = ('/some/cert', '/some/key')
proxies = {'https': 'https://proxy.com'}
proxies_config = {'proxy_client_cert': "path/to/cert"}
with patch('botocore.httpsession.create_urllib3_context'):
session = URLLib3Session(proxies=proxies,
client_cert=cert,
proxies_config=proxies_config)
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(proxies['https'],
proxy_headers={},
cert_file=cert[0],
key_file=cert[1],
proxy_ssl_context=ANY)
self.assert_request_sent()
def __init__(self, aws_lambda_config, internal_storage):
"""
Initialize AWS Lambda Backend
"""
logger.debug('Creating AWS Lambda client')
self.name = 'aws_lambda'
self.type = 'faas'
self.aws_lambda_config = aws_lambda_config
self.internal_storage = internal_storage
self.user_agent = aws_lambda_config['user_agent']
self.user_key = aws_lambda_config['access_key_id'][-4:]
self.package = 'lithops_v{}_{}'.format(lithops.__version__,
self.user_key.lower()).replace(
'.', '-')
self.region_name = aws_lambda_config['region_name']
self.role_arn = aws_lambda_config['execution_role']
logger.debug('Creating Boto3 AWS Session and Lambda Client')
self.aws_session = boto3.Session(
aws_access_key_id=aws_lambda_config['access_key_id'],
aws_secret_access_key=aws_lambda_config['secret_access_key'],
region_name=self.region_name)
self.lambda_client = self.aws_session.client(
'lambda',
region_name=self.region_name,
config=botocore.client.Config(user_agent_extra=self.user_agent))
self.credentials = self.aws_session.get_credentials()
self.session = URLLib3Session()
self.host = f'lambda.{self.region_name}.amazonaws.com'
if self.aws_lambda_config['account_id']:
self.account_id = self.aws_lambda_config['account_id']
else:
sts_client = self.aws_session.client('sts',
region_name=self.region_name)
self.account_id = sts_client.get_caller_identity()["Account"]
self.ecr_client = self.aws_session.client('ecr',
region_name=self.region_name)
msg = COMPUTE_CLI_MSG.format('AWS Lambda')
logger.info("{} - Region: {}".format(msg, self.region_name))
def head_index_from_es(index_name, method='HEAD', proto='https://'):
es_url = urlparse.urlparse(ES_ENDPOINT)
es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT
'''Post data to ES endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method,
url=proto + es_endpoint + '/' + urllib.quote(index_name),
headers={'Host': es_endpoint})
es_region = ES_REGION or os.environ['AWS_REGION']
session = Session()
SigV4Auth(get_credentials(session), 'es',
os.environ['AWS_REGION']).add_auth(req)
http_session = URLLib3Session()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
logger.info('Index %s do exists, continue update setting', index_name)
return True
else:
logger.warning('Index %s does not exists, need to create.', index_name)
return False
def test_proxies_config_settings(self):
proxies = {'http': 'http://proxy.com'}
proxies_config = {
'proxy_ca_bundle': 'path/to/bundle',
'proxy_client_cert': ('path/to/cert', 'path/to/key'),
'proxy_use_forwarding_for_https': False,
}
use_forwarding = proxies_config['proxy_use_forwarding_for_https']
with patch('botocore.httpsession.create_urllib3_context'):
session = URLLib3Session(proxies=proxies,
proxies_config=proxies_config)
self.request.url = 'http://example.com/'
session.send(self.request.prepare())
self.assert_proxy_manager_call(
proxies['http'],
proxy_headers={},
proxy_ssl_context=ANY,
use_forwarding_for_https=use_forwarding)
self.assert_request_sent(url=self.request.url)
def put_data_to_es(payload, path, method='PUT', proto='https://'):
es_url = urlparse.urlparse(ES_ENDPOINT)
es_endpoint = es_url.netloc or es_url.path # Extract the domain name in ES_ENDPOINT
'''Post data to ES endpoint with SigV4 signed http headers'''
req = AWSRequest(method=method,
url=proto + es_endpoint + '/' + urllib.quote(path),
data=payload,
headers={
'Host': es_endpoint,
'Content-Type': 'application/json'
})
es_region = ES_REGION or os.environ['AWS_REGION']
session = Session()
SigV4Auth(get_credentials(session), 'es',
os.environ['AWS_REGION']).add_auth(req)
http_session = URLLib3Session()
res = http_session.send(req.prepare())
if res.status_code >= 200 and res.status_code <= 299:
return res._content
else:
raise ES_Exception(res.status_code, res._content)
def test_basic_streaming_request(self):
session = URLLib3Session()
self.request.stream_output = True
session.send(self.request.prepare())
self.assert_request_sent()
self.response.stream.assert_not_called()
def test_chunked_encoding_is_not_set_without_header(self):
session = URLLib3Session()
session.send(self.request.prepare())
self.assert_request_sent(chunked=False)
def test_basic_https_request(self):
session = URLLib3Session()
self.request.url = 'https://example.com/'
session.send(self.request.prepare())
self.assert_request_sent()
def make_request_with_error(self, error):
self.connection.urlopen.side_effect = error
session = URLLib3Session()
session.send(self.request.prepare())
def test_ssl_context_is_explicit(self):
session = URLLib3Session()
session.send(self.request.prepare())
_, manager_kwargs = self.pool_manager_cls.call_args
self.assertIsNotNone(manager_kwargs.get('ssl_context'))
def test_session_forwards_socket_options_to_pool_manager(self):
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
URLLib3Session(socket_options=socket_options)
self.assert_pool_manager_call(socket_options=socket_options)
def test_proxy_request_ssl_context_is_explicit(self):
proxies = {'http': 'http://proxy.com'}
session = URLLib3Session(proxies=proxies)
session.send(self.request.prepare())
_, proxy_kwargs = self.proxy_manager_fun.call_args
self.assertIsNotNone(proxy_kwargs.get('ssl_context'))