def test_no_profile_checks_env_provider(self):
self.config['profile'] = None
self.session._profile = None
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(
any(isinstance(p, EnvProvider) for p in resolver.providers))
def test_no_profile_checks_env_provider(self):
self.config['profile'] = None
self.session.profile = None
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(
any(isinstance(p, EnvProvider) for p in resolver.providers))
def test_no_profile_checks_env_provider(self):
# If no profile is provided,
self.session_instance_vars.pop('profile', None)
resolver = credentials.create_credential_resolver(self.session)
# Then an EnvProvider should be part of our credential lookup chain.
self.assertTrue(
any(isinstance(p, EnvProvider) for p in resolver.providers))
def test_no_profile_checks_env_provider(self):
# If no profile is provided,
self.session_instance_vars.pop('profile', None)
resolver = credentials.create_credential_resolver(self.session)
# Then an EnvProvider should be part of our credential lookup chain.
self.assertTrue(
any(isinstance(p, EnvProvider) for p in resolver.providers))
def __init__(self,
name,
access_key=None,
secret_key=None,
security_token=None,
profile_name=None,
**kwargs):
"""
Create a new BotoCredentialAdapter.
"""
# TODO: We take kwargs because new boto2 versions have an 'anon'
# argument and we want to be future proof
if (name == 'aws'
or name is None) and access_key is None and not kwargs.get(
'anon', False):
# We are on AWS and we don't have credentials passed along and we aren't anonymous.
# We will backend into a boto3 resolver for getting credentials.
# Make sure to enable boto3's own caching, so we can share that
# cash with pure boto3 code elsewhere in Toil.
self._boto3_resolver = create_credential_resolver(
Session(profile=profile_name), cache=JSONFileCache())
else:
# We will use the normal flow
self._boto3_resolver = None
# Pass along all the arguments
super(BotoCredentialAdapter,
self).__init__(name,
access_key=access_key,
secret_key=secret_key,
security_token=security_token,
profile_name=profile_name,
**kwargs)
def send_to_es(self, path, method="GET", payload={}):
"""Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
Args:
path (str): path to send to ES
method (str, optional): HTTP method default:GET
payload (dict, optional): additional payload used during POST or PUT
Returns:
dict: json answer converted in dict
Raises:
#: Error during ES communication
ES_Exception: Description
"""
if not path.startswith("/"):
path = "/" + path
es_region = self.cfg["es_endpoint"].split(".")[1]
headers = {
"Host": self.cfg["es_endpoint"],
"Content-Type": "application/json"
}
# send to ES with exponential backoff
retries = 0
while retries < int(self.cfg["es_max_retry"]):
if retries > 0:
seconds = (2**retries) * .1
time.sleep(seconds)
req = AWSRequest(
method=method,
url="https://{}{}".format(
self.cfg["es_endpoint"], quote(path)),
data=json.dumps(payload),
params={"format": "json"},
headers=headers)
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
SigV4Auth(credentials, 'es', es_region).add_auth(req)
try:
preq = req.prepare()
session = Session()
res = session.send(preq)
if res.status_code >= 200 and res.status_code <= 299:
return json.loads(res.content)
else:
raise ES_Exception(res.status_code, res._content)
except ES_Exception as e:
if (e.status_code >= 500) and (e.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def test_create_credential_resolver(self):
fake_session = mock.Mock()
config = {
'credentials_file': 'a',
'legacy_config_file': 'b',
'config_file': 'c',
'metadata_service_timeout': 'd',
'metadata_service_num_attempts': 'e',
}
fake_session.get_config_variable = lambda x: config[x]
resolver = credentials.create_credential_resolver(fake_session)
def __init__(self):
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
awsauth = AWS4Auth(
credentials.access_key,
credentials.secret_key,
'us-east-1',
'es',
session_token=credentials.token)
self.es = Elasticsearch(
hosts=[{'host': self.es_endpoint, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
def test_assume_role_uses_correct_region(self):
config = ('[profile A]\n'
'role_arn = arn:aws:iam::123456789:role/RoleA\n'
'source_profile = B\n\n'
'[profile B]\n'
'aws_access_key_id = abc123\n'
'aws_secret_access_key = def456\n')
self.write_config(config)
session = Session(profile='A')
# Verify that when we configure the session with a specific region
# that we use that region when creating the sts client.
session.set_config_variable('region', 'cn-north-1')
create_client, expected_creds = self.create_stubbed_sts_client(session)
session.create_client = create_client
resolver = create_credential_resolver(session)
provider = resolver.get_provider('assume-role')
creds = provider.load()
self.assert_creds_equal(creds, expected_creds)
self.assertEqual(self.actual_client_region, 'cn-north-1')
def test_assume_role_uses_correct_region(self):
config = (
'[profile A]\n'
'role_arn = arn:aws:iam::123456789:role/RoleA\n'
'source_profile = B\n\n'
'[profile B]\n'
'aws_access_key_id = abc123\n'
'aws_secret_access_key = def456\n'
)
self.write_config(config)
session = Session(profile='A')
# Verify that when we configure the session with a specific region
# that we use that region when creating the sts client.
session.set_config_variable('region', 'cn-north-1')
create_client, expected_creds = self.create_stubbed_sts_client(session)
session.create_client = create_client
resolver = create_credential_resolver(session)
provider = resolver.get_provider('assume-role')
creds = provider.load()
self.assert_creds_equal(creds, expected_creds)
self.assertEqual(self.actual_client_region, 'cn-north-1')
def send_to_es(self, path, method="GET", payload={}):
"""Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
Args:
path (str): path to send to ES
method (str, optional): HTTP method default:GET
payload (dict, optional): additional payload used during POST or PUT
Returns:
dict: json answer converted in dict
Raises:
#: Error during ES communication
ESException: Description
"""
if not path.startswith("/"):
path = "/" + path
es_region = self.cfg["es_endpoint"].split(".")[1]
req = AWSRequest(method=method,
url="https://%s%s?pretty&format=json" %
(self.cfg["es_endpoint"], quote(path)),
data=payload,
headers={'Host': self.cfg["es_endpoint"]})
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
SigV4Auth(credentials, 'es', es_region).add_auth(req)
preq = req.prepare()
session = Session()
res = session.send(preq)
session.close()
if res.status_code >= 200 and res.status_code <= 299:
return json.loads(res.content)
else:
raise ESException(res.status_code, res._content)
def test_env_provider_added_if_profile_from_env_set(self):
self.fake_env_vars['profile'] = 'profile-from-env'
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(
any(isinstance(p, EnvProvider) for p in resolver.providers))
def test_explicit_profile_ignores_env_provider(self):
self.session_instance_vars['profile'] = 'dev'
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(
all(not isinstance(p, EnvProvider) for p in resolver.providers))
def send_to_es(self,
path="/",
method=None,
payload=None,
extra_headers=None):
"""Low-level POST data to Amazon Elasticsearch Service generating a Sigv4 signed request
Args:
path (str): path to send to ES
method (str, optional): HTTP method default:GET
payload (dict, optional): additional payload used during POST or PUT
Returns:
dict: json answer converted in dict
Raises:
#: Error during ES communication
ESException: Description
"""
# resolve default kwargs
payload = payload or {}
extra_headers = extra_headers or {}
if not path.startswith("/"):
path = "/" + path
method = method if method else self.method
es_region = self.cfg["es_endpoint"].split(".")[1]
# send to ES with exponential backoff
retries = 0
while retries < int(self.cfg["es_max_retry"]):
if retries > 0:
seconds = (2**retries) * .1
# print('Waiting for %.1f seconds', seconds)
time.sleep(seconds)
extra_headers.update({"Host": self.cfg["es_endpoint"]})
req = AWSRequest(
method=method,
url=('https://%s%s?pretty&format=json' %
(self.cfg['es_endpoint'], urllib.parse.quote(path))),
data=payload,
headers=extra_headers)
credential_resolver = create_credential_resolver(get_session())
credentials = credential_resolver.load_credentials()
SigV4Auth(credentials, 'es', es_region).add_auth(req)
try:
preq = req.prepare()
session = Session()
res = session.send(preq)
if res.status_code >= 200 and res.status_code <= 299:
LOGGER.debug("%s %s", res.status_code, res.content)
return json.loads(res.content)
else:
LOGGER.debug("%s %s", res.status_code, res.content)
raise ESException(res.status_code, res.content)
except ESException as err:
if (err.status_code >= 500) and (err.status_code <= 599):
retries += 1 # Candidate for retry
else:
raise # Stop retrying, re-raise exception
def test_no_profile_env_provider_is_first(self):
self.config['profile'] = None
self.session.profile = None
resolver = credentials.create_credential_resolver(self.session)
self.assertIsInstance(resolver.providers[0], credentials.EnvProvider)
def test_create_credential_resolver(self):
resolver = credentials.create_credential_resolver(self.session)
self.assertIsInstance(resolver, credentials.CredentialResolver)
def test_no_profile_env_provider_is_first(self):
self.config['profile'] = None
self.session._profile = None
resolver = credentials.create_credential_resolver(self.session)
self.assertIsInstance(resolver.providers[0], credentials.EnvProvider)
def test_explicit_profile_ignores_env_provider(self):
self.config['profile'] = 'dev'
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(
all(not isinstance(p, EnvProvider) for p in resolver.providers))
def test_create_credential_resolver(self):
resolver = credentials.create_credential_resolver(self.session)
self.assertIsInstance(resolver, credentials.CredentialResolver)
def test_env_provider_added_if_profile_from_env_set(self):
self.fake_env_vars['profile'] = 'profile-from-env'
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(
any(isinstance(p, EnvProvider) for p in resolver.providers))
def test_explicit_profile_ignores_env_provider(self):
self.session_instance_vars["profile"] = "dev"
resolver = credentials.create_credential_resolver(self.session)
self.assertTrue(all(not isinstance(p, EnvProvider) for p in resolver.providers))