def test_old_model_continues_to_work():
# This test ensures that ibm_botocore can load the service models as they exist
# today. There's a directory in tests/functional/models that is a
# snapshot of a service model. This test ensures that we can continue
# to stub an API call using this model. That way if the models ever
# change we have a mechanism to ensure that the existing models continue
# to work with ibm_botocore. The test should not change (with the exception
# of potential changes to the ClientHTTPStubber), and the files in
# tests/functional/models should not change!
session = Session()
loader = session.get_component('data_loader')
# We're adding our path to the existing search paths so we don't have to
# copy additional data files such as _retry.json to our FIXED_MODELS_DIR.
# We only care about the service model and endpoints file not changing.
# This also prevents us from having to make any changes to this models dir
# if we end up adding a new data file that's needed to create clients.
# We're adding our FIXED_MODELS_DIR as the first element in the list to
# ensure we load the endpoints.json file from FIXED_MODELS_DIR. For the
# service model we have an extra safety net where we can choose a custom
# client name.
loader.search_paths.insert(0, FIXED_MODELS_DIR)
# The model dir we copied was renamed to 'custom-lambda'
# to ensure we're loading our version of the model and not
# the built in one.
client = session.create_client(
'custom-acm',
region_name='us-west-2',
aws_access_key_id='foo',
aws_secret_access_key='bar',
)
with ClientHTTPStubber(client) as stubber:
stubber.add_response(url='https://acm.us-west-2.amazonaws.com/',
headers={
'x-amzn-RequestId': 'abcd',
'Date': 'Fri, 26 Oct 2018 01:46:30 GMT',
'Content-Length': '29',
'Content-Type': 'application/x-amz-json-1.1'
},
body=b'{"CertificateSummaryList":[]}')
response = client.list_certificates()
assert_equal(
response, {
'CertificateSummaryList': [],
'ResponseMetadata': {
'HTTPHeaders': {
'content-length': '29',
'content-type': 'application/x-amz-json-1.1',
'date': 'Fri, 26 Oct 2018 01:46:30 GMT',
'x-amzn-requestid': 'abcd'
},
'HTTPStatusCode': 200,
'RequestId': 'abcd',
'RetryAttempts': 0
}
})
# Also verify we can use the paginators as well.
assert_equal(client.can_paginate('list_certificates'), True)
assert_equal(client.waiter_names, ['certificate_validated'])
def setUp(self):
self.env_original = os.environ.copy()
self.environ_copy = os.environ.copy()
super(TestAssumeRoleCredentials, self).setUp()
os.environ = self.environ_copy
# The tests rely on manipulating AWS_CONFIG_FILE,
# but we also need to make sure we don't accidentally
# pick up the ~/.aws/credentials file either.
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = str(uuid4())
self.parent_session = Session()
self.iam = self.parent_session.create_client('iam')
self.sts = self.parent_session.create_client('sts')
self.tempdir = tempfile.mkdtemp()
self.config_file = os.path.join(self.tempdir, 'config')
# A role trust policy that allows the current account to call assume
# role on itself.
account_id = self.sts.get_caller_identity()['Account']
self.role_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::%s:root" % account_id
},
"Action": "sts:AssumeRole"
}]
}
def test_credential_process_returns_error(self):
config = ('[profile processcreds]\n'
'credential_process = %s --raise-error\n')
config = config % self.credential_process
with temporary_file('w') as f:
f.write(config)
f.flush()
self.environ['AWS_CONFIG_FILE'] = f.name
session = Session(profile='processcreds')
# This regex validates that there is no substring: b'
# The reason why we want to validate that is that we want to
# make sure that stderr is actually decoded so that in
# exceptional cases the error is properly formatted.
# As for how the regex works:
# `(?!b').` is a negative lookahead, meaning that it will only
# match if it is not followed by the pattern `b'`. Since it is
# followed by a `.` it will match any character not followed by
# that pattern. `((?!hede).)*` does that zero or more times. The
# final pattern adds `^` and `$` to anchor the beginning and end
# of the string so we can know the whole string is consumed.
# Finally `(?s)` at the beginning makes dots match newlines so
# we can handle a multi-line string.
reg = r"(?s)^((?!b').)*$"
with self.assertRaisesRegexp(CredentialRetrievalError, reg):
session.get_credentials()
def test_public_apis_will_not_be_signed():
session = Session()
# Mimic the scenario that user does not have aws credentials setup
session.get_credentials = mock.Mock(return_value=None)
for service_name in PUBLIC_API_TESTS:
client = session.create_client(service_name, REGIONS[service_name])
for operation_name in PUBLIC_API_TESTS[service_name]:
kwargs = PUBLIC_API_TESTS[service_name][operation_name]
method = getattr(client, xform_name(operation_name))
yield (_test_public_apis_will_not_be_signed, method, kwargs)
def test_tagged_union_member_name_does_not_coincide_with_unknown_key(self):
# This test ensures that operation models do not use SDK_UNKNOWN_MEMBER
# as a member name. Thereby reserving SDK_UNKNOWN_MEMBER for the parser to
# set as a key on the reponse object. This is necessary when the client
# encounters a member that it is unaware of or not modeled.
session = Session()
for service_name in session.get_available_services():
service_model = session.get_service_model(service_name)
for shape_name in service_model.shape_names:
shape = service_model.shape_for(shape_name)
if hasattr(shape, 'is_tagged_union') and shape.is_tagged_union:
self.assertNotIn('SDK_UNKNOWN_MEMBER', shape.members)
def test_honors_aws_shared_credentials_file_env_var(self):
with temporary_file('w') as f:
f.write('[default]\n'
'aws_access_key_id=custom1\n'
'aws_secret_access_key=custom2\n')
f.flush()
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = f.name
s = Session()
credentials = s.get_credentials()
self.assertEqual(credentials.access_key, 'custom1')
self.assertEqual(credentials.secret_key, 'custom2')
def test_paginators_and_waiters_are_not_lost_in_new_version():
for service_name in Session().get_available_services():
versions = Loader().list_api_versions(service_name, 'service-2')
if len(versions) > 1:
for type_name in ['paginators-1', 'waiters-2']:
yield (_test_model_is_not_lost, service_name, type_name,
versions[-2], versions[-1])
def test_assume_role_with_credential_source(self):
# Create a role with read access to S3
role = self.create_role(self.role_policy, S3_READ_POLICY_ARN)
# Create a user that can assume the role and get static credentials
# for it.
user_policy_arn = self.create_assume_policy(role['Arn'])
user = self.create_user([user_policy_arn])
user_creds = self.create_creds(user['UserName'])
# Setup the config file with the profile we'll be using.
config = ('[profile assume]\n'
'role_arn = %s\n'
'credential_source = Environment\n')
config = config % role['Arn']
with open(self.config_file, 'w') as f:
f.write(config)
# Wait for IAM permissions to propagate
self.wait_for_assume_role(
role_arn=role['Arn'],
access_key=user_creds['AccessKeyId'],
secret_key=user_creds['SecretAccessKey'],
)
# Setup the environment so that our new config file is THE config
# file and add the expected credentials since we're using the
# environment as our credential source.
os.environ['AWS_CONFIG_FILE'] = self.config_file
os.environ['AWS_SECRET_ACCESS_KEY'] = user_creds['SecretAccessKey']
os.environ['AWS_ACCESS_KEY_ID'] = user_creds['AccessKeyId']
self.assert_s3_read_only_session(Session(profile='assume'))
def test_assume_role_web_identity_uses_same_region_as_client(self):
token_file = os.path.join(self.tempdir, 'token.jwt')
with open(token_file, 'w') as f:
f.write('some-token')
config = ('[profile A]\n'
'sts_regional_endpoints = regional\n'
'role_arn = arn:aws:iam::123456789:role/RoleA\n'
'web_identity_token_file = %s\n'
'source_profile = B\n\n'
'[profile B]\n'
'aws_access_key_id = abc123\n'
'aws_secret_access_key = def456\n' % token_file)
self.write_config(config)
# Make an arbitrary client and API call as we are really only
# looking to make sure the STS assume role call uses the correct
# endpoint.
session = Session(profile='A')
with SessionHTTPStubber(session) as stubber:
self.add_assume_role_with_web_identity_http_response(stubber)
# Make an arbitrary client and API call as we are really only
# looking to make sure the STS assume role call uses the correct
# endpoint.
self.make_stubbed_client_call_to_region(session, stubber,
'us-west-2')
self.assertEqual(stubber.requests[0].url,
'https://sts.us-west-2.amazonaws.com/')
def test_credential_process(self):
config = ('[profile processcreds]\n' 'credential_process = %s\n')
config = config % self.credential_process
with temporary_file('w') as f:
f.write(config)
f.flush()
self.environ['AWS_CONFIG_FILE'] = f.name
credentials = Session(profile='processcreds').get_credentials()
self.assertEqual(credentials.access_key, 'spam')
self.assertEqual(credentials.secret_key, 'eggs')
def test_assume_role_uses_correct_region(self):
config = ('[profile A]\n'
'role_arn = arn:aws:iam::123456789:role/RoleA\n'
'source_profile = B\n\n'
'[profile B]\n'
'aws_access_key_id = abc123\n'
'aws_secret_access_key = def456\n')
self.write_config(config)
session = Session(profile='A')
# Verify that when we configure the session with a specific region
# that we use that region when creating the sts client.
session.set_config_variable('region', 'cn-north-1')
create_client, expected_creds = self.create_stubbed_sts_client(session)
session.create_client = create_client
resolver = create_credential_resolver(session)
provider = resolver.get_provider('assume-role')
creds = provider.load()
self.assert_creds_equal(creds, expected_creds)
self.assertEqual(self.actual_client_region, 'cn-north-1')
def create_session(self, *args, **kwargs):
"""
Create a new session with the given arguments. Additionally,
this method will set the credentials file to the test credentials
used by the following test cases.
"""
kwargs['session_vars'] = {
'credentials_file': (None, None,
os.path.join(os.path.dirname(__file__),
'test-credentials'), None)
}
return Session(*args, **kwargs)
def test_recursive_assume_role(self):
# Create the final role, the one that will actually have access to s3
final_role = self.create_role(self.role_policy, S3_READ_POLICY_ARN)
# Create the role that can assume the final role
middle_policy_arn = self.create_assume_policy(final_role['Arn'])
middle_role = self.create_role(self.role_policy, middle_policy_arn)
# Create a user that can only assume the middle-man role, and then get
# static credentials for it.
user_policy_arn = self.create_assume_policy(middle_role['Arn'])
user = self.create_user([user_policy_arn])
user_creds = self.create_creds(user['UserName'])
# Setup the config file with the profiles we'll be using. For
# convenience static credentials are placed here instead of putting
# them in the credentials file.
config = ('[default]\n'
'aws_access_key_id = %s\n'
'aws_secret_access_key = %s\n'
'[profile middle]\n'
'source_profile = default\n'
'role_arn = %s\n'
'[profile final]\n'
'source_profile = middle\n'
'role_arn = %s\n')
config = config % (user_creds['AccessKeyId'],
user_creds['SecretAccessKey'], middle_role['Arn'],
final_role['Arn'])
with open(self.config_file, 'w') as f:
f.write(config)
# Wait for IAM permissions to propagate
middle_creds = self.wait_for_assume_role(
role_arn=middle_role['Arn'],
access_key=user_creds['AccessKeyId'],
secret_key=user_creds['SecretAccessKey'],
)
self.wait_for_assume_role(
role_arn=final_role['Arn'],
access_key=middle_creds['AccessKeyId'],
secret_key=middle_creds['SecretAccessKey'],
token=middle_creds['SessionToken'],
)
# Configure our credentials file to be THE credentials file
os.environ['AWS_CONFIG_FILE'] = self.config_file
self.assert_s3_read_only_session(Session(profile='final'))
class TestAssumeRoleCredentials(BaseEnvVar):
def setUp(self):
self.env_original = os.environ.copy()
self.environ_copy = os.environ.copy()
super(TestAssumeRoleCredentials, self).setUp()
os.environ = self.environ_copy
# The tests rely on manipulating AWS_CONFIG_FILE,
# but we also need to make sure we don't accidentally
# pick up the ~/.aws/credentials file either.
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = str(uuid4())
self.parent_session = Session()
self.iam = self.parent_session.create_client('iam')
self.sts = self.parent_session.create_client('sts')
self.tempdir = tempfile.mkdtemp()
self.config_file = os.path.join(self.tempdir, 'config')
# A role trust policy that allows the current account to call assume
# role on itself.
account_id = self.sts.get_caller_identity()['Account']
self.role_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::%s:root" % account_id
},
"Action": "sts:AssumeRole"
}]
}
def tearDown(self):
super(TestAssumeRoleCredentials, self).tearDown()
shutil.rmtree(self.tempdir)
os.environ = self.env_original.copy()
def random_name(self):
return 'botocoretest-' + random_chars(10)
def create_role(self, policy_document, policy_arn=None):
name = self.random_name()
response = self.iam.create_role(
RoleName=name,
AssumeRolePolicyDocument=json.dumps(policy_document))
self.addCleanup(self.iam.delete_role, RoleName=name)
if policy_arn:
self.iam.attach_role_policy(RoleName=name, PolicyArn=policy_arn)
self.addCleanup(self.iam.detach_role_policy,
RoleName=name,
PolicyArn=policy_arn)
return response['Role']
def create_user(self, policy_arns):
name = self.random_name()
user = self.iam.create_user(UserName=name)['User']
self.addCleanup(self.iam.delete_user, UserName=name)
for arn in policy_arns:
self.iam.attach_user_policy(UserName=name, PolicyArn=arn)
self.addCleanup(self.iam.detach_user_policy,
UserName=name,
PolicyArn=arn)
return user
def create_creds(self, user_name):
creds = self.iam.create_access_key(UserName=user_name)['AccessKey']
self.addCleanup(self.iam.delete_access_key,
UserName=user_name,
AccessKeyId=creds['AccessKeyId'])
return creds
def wait_for_assume_role(self,
role_arn,
access_key,
secret_key,
token=None,
attempts=30,
delay=10,
success_delay=1,
num_success=4):
for _ in range(num_success):
creds = self._wait_for_assume_role(role_arn, access_key,
secret_key, token, attempts,
delay)
time.sleep(success_delay)
return creds
def _wait_for_assume_role(self, role_arn, access_key, secret_key, token,
attempts, delay):
# "Why not use the policy simulator?" you might ask. The answer is
# that the policy simulator will return success far before you can
# actually make the calls.
client = self.parent_session.create_client(
'sts',
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
aws_session_token=token)
attempts_remaining = attempts
role_session_name = random_chars(10)
while attempts_remaining > 0:
attempts_remaining -= 1
try:
result = client.assume_role(RoleArn=role_arn,
RoleSessionName=role_session_name)
return result['Credentials']
except ClientError as e:
code = e.response.get('Error', {}).get('Code')
if code in ["InvalidClientTokenId", "AccessDenied"]:
time.sleep(delay)
else:
raise
raise Exception("Unable to assume role %s" % role_arn)
def create_assume_policy(self, role_arn):
policy_document = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": role_arn,
"Action": "sts:AssumeRole"
}]
}
name = self.random_name()
response = self.iam.create_policy(
PolicyName=name, PolicyDocument=json.dumps(policy_document))
self.addCleanup(self.iam.delete_policy,
PolicyArn=response['Policy']['Arn'])
return response['Policy']['Arn']
def assert_s3_read_only_session(self, session):
# Calls to S3 should succeed
s3 = session.create_client('s3')
s3.list_buckets()
# Calls to other services should not
iam = session.create_client('iam')
try:
iam.list_groups()
self.fail("Expected call to list_groups to fail, but it passed.")
except ClientError as e:
code = e.response.get('Error', {}).get('Code')
if code != 'AccessDenied':
raise
def test_recursive_assume_role(self):
# Create the final role, the one that will actually have access to s3
final_role = self.create_role(self.role_policy, S3_READ_POLICY_ARN)
# Create the role that can assume the final role
middle_policy_arn = self.create_assume_policy(final_role['Arn'])
middle_role = self.create_role(self.role_policy, middle_policy_arn)
# Create a user that can only assume the middle-man role, and then get
# static credentials for it.
user_policy_arn = self.create_assume_policy(middle_role['Arn'])
user = self.create_user([user_policy_arn])
user_creds = self.create_creds(user['UserName'])
# Setup the config file with the profiles we'll be using. For
# convenience static credentials are placed here instead of putting
# them in the credentials file.
config = ('[default]\n'
'aws_access_key_id = %s\n'
'aws_secret_access_key = %s\n'
'[profile middle]\n'
'source_profile = default\n'
'role_arn = %s\n'
'[profile final]\n'
'source_profile = middle\n'
'role_arn = %s\n')
config = config % (user_creds['AccessKeyId'],
user_creds['SecretAccessKey'], middle_role['Arn'],
final_role['Arn'])
with open(self.config_file, 'w') as f:
f.write(config)
# Wait for IAM permissions to propagate
middle_creds = self.wait_for_assume_role(
role_arn=middle_role['Arn'],
access_key=user_creds['AccessKeyId'],
secret_key=user_creds['SecretAccessKey'],
)
self.wait_for_assume_role(
role_arn=final_role['Arn'],
access_key=middle_creds['AccessKeyId'],
secret_key=middle_creds['SecretAccessKey'],
token=middle_creds['SessionToken'],
)
# Configure our credentials file to be THE credentials file
os.environ['AWS_CONFIG_FILE'] = self.config_file
self.assert_s3_read_only_session(Session(profile='final'))
def test_assume_role_with_credential_source(self):
# Create a role with read access to S3
role = self.create_role(self.role_policy, S3_READ_POLICY_ARN)
# Create a user that can assume the role and get static credentials
# for it.
user_policy_arn = self.create_assume_policy(role['Arn'])
user = self.create_user([user_policy_arn])
user_creds = self.create_creds(user['UserName'])
# Setup the config file with the profile we'll be using.
config = ('[profile assume]\n'
'role_arn = %s\n'
'credential_source = Environment\n')
config = config % role['Arn']
with open(self.config_file, 'w') as f:
f.write(config)
# Wait for IAM permissions to propagate
self.wait_for_assume_role(
role_arn=role['Arn'],
access_key=user_creds['AccessKeyId'],
secret_key=user_creds['SecretAccessKey'],
)
# Setup the environment so that our new config file is THE config
# file and add the expected credentials since we're using the
# environment as our credential source.
os.environ['AWS_CONFIG_FILE'] = self.config_file
os.environ['AWS_SECRET_ACCESS_KEY'] = user_creds['SecretAccessKey']
os.environ['AWS_ACCESS_KEY_ID'] = user_creds['AccessKeyId']
self.assert_s3_read_only_session(Session(profile='assume'))
def _paginators_and_waiters_test_cases():
for service_name in Session().get_available_services():
versions = Loader().list_api_versions(service_name, 'service-2')
if len(versions) > 1:
for type_name in ['paginators-1', 'waiters-2']:
yield service_name, type_name, versions[-2], versions[-1]