def _test_lambda_write_cloud_watch__with_asserts(self):
group_name = '/unit-tests/test_log_group'
stream_name = Misc.random_string_and_numbers(prefix='tmp_stream_')
message = 'this is a message sent from an lambda function'
lambda_name = 'osbot_aws.lambdas.dev.write_cloud_watch_log'
log_group_arn = 'arn:aws:logs:eu-west-2:244560807427:log-group:{0}*'.format(
group_name)
policy_name = 'temp_policy_for_lambda_write_cloud_watch'
role_name = 'temp_role_for_lambda_invocation'
policy_actions = ['logs:PutLogEvents']
logs = Logs(group_name=group_name, stream_name=stream_name)
logs.group_create()
logs.stream_create()
iam_role = IAM(role_name=role_name)
iam_policy = IAM_Policy(policy_name=policy_name)
iam_policy.add_statement_allow(policy_actions, [log_group_arn])
policy_arn = iam_policy.create(
delete_before_create=True).get('policy_arn')
assert iam_policy.exists() is True
assert iam_role.role_exists() is True
assert logs.group_exists() is True
assert logs.stream_exists() is True
assert set(iam_role.role_policies()) == {
'AWSXrayWriteOnlyAccess', 'policy_temp_role_for_lambda_invocation'
}
iam_role.role_policy_attach(policy_arn)
assert set(iam_role.role_policies()) == {
'AWSXrayWriteOnlyAccess', 'policy_temp_role_for_lambda_invocation',
'temp_policy_for_lambda_write_cloud_watch'
}
sleep(10) # wait for AWS to propagate role update
payload = {
'group_name': group_name,
'stream_name': stream_name,
'message': message
}
lambda_obj = Lambda_Package(lambda_name) #.update_with_root_folder()
result = lambda_obj.invoke(payload)
sleep(1) # wait for Cloudwatch to update
assert result.get('status') == 'ok'
assert logs.messages() == [message]
assert iam_policy.delete() is True
assert logs.group_delete() is True
assert logs.group_exists() is False
assert set(iam_role.role_policies()) == {
'AWSXrayWriteOnlyAccess', 'policy_temp_role_for_lambda_invocation'
}
def test_role_policies(self):
policies = self.iam.role_policies()
assert len(set(policies)) == 0
iam = IAM(role_name='AWSServiceRoleForAPIGateway')
assert iam.role_policies() == {'APIGatewayServiceRolePolicy': 'arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy'}
assert len(iam.role_policies_statements().get('APIGatewayServiceRolePolicy')[0].get('Action')) > 10
def test_role_policies(self):
iam = IAM(role_name='AWSBatchServiceRole')
assert iam.role_policies() == {
'AWSBatchServiceRole':
'arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole'
}
assert len(iam.role_policies_statements().get('AWSBatchServiceRole')
[0].get('Action')) == 59
class Test_IAM(Test_Helper):
@classmethod
def setUpClass(cls):
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
iam = IAM(user_name=test_user, role_name=test_role)
if iam.user_exists() is False:
iam.user_create()
if iam.role_exists() is False:
iam.role_create(policy_document)
@classmethod
def tearDownClass(cls):
if delete_created:
iam = IAM(user_name=test_user,role_name=test_role)
iam.user_delete()
assert iam.user_exists() is False
iam.role_delete()
assert iam.role_exists() is False
assert iam.role_arn() is None
def setUp(self):
super().setUp()
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
self.iam = IAM(user_name=test_user,role_name=test_role )
# ------ tests ------
@unittest.skip("Doesn't work in CodeBuild since there is only one configuration in there")
def test_account_id(self):
account_id_1 = self.iam.account_id('gs-detect-aws') # todo: rewrite since account_id doesn't take this parameter any more
assert AWS_Config().aws_session_profile_name() == 'gs-detect-aws'
self.iam._account_id = None
self.iam._sts = None
account_id_2 = self.iam.account_id('default')
assert AWS_Config().aws_session_profile_name() == 'default'
assert account_id_1 != account_id_2
self.iam._account_id = None
self.iam._sts = None
account_id_3 = self.iam.account_id()
assert AWS_Config().aws_session_profile_name() == 'default'
assert account_id_2 == account_id_3
@pytest.mark.skip('Fix test')
def test_access_keys(self):
assert len(self.iam.access_keys(index_by='AccessKeyId')) > 0
assert len(self.iam.access_keys(group_by='UserName' )) > 0
def test_caller_identity(self):
assert set(self.iam.caller_identity()) == {'UserId', 'Account', 'Arn'}
@pytest.mark.skip('Fix test')
def test_groups(self):
assert len(self.iam.groups()) > 5
def test_policies(self):
assert len(self.iam.policies()) > 500
def test_policy_arn(self):
assert self.iam.policy_arn('aaa' ) == 'arn:aws:iam::{0}:policy/aaa' .format(account_id)
assert self.iam.policy_arn('aaa/bbb' ) == 'arn:aws:iam::{0}:policy/aaa/bbb'.format(account_id)
assert self.iam.policy_arn('aa','/bb' ) == 'arn:aws:iam::{0}:policy/bb/aa' .format(account_id)
assert self.iam.policy_arn('aa','/bb','cc') == 'arn:aws:iam::cc:policy/bb/aa'
assert self.iam.policy_arn(None) is None
def test_policy_create__policy_delete__policy_details(self):
policy_name = 'test_policy'
self.iam.policy_delete_by_name(policy_name)
new_policy_document = { "Version": "2012-10-17",
"Statement": [ { "Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*" }]}
result = self.iam.policy_create(policy_name, new_policy_document)
expected_policy_arn = self.iam.policy_arn(policy_name)
status = result.get('status')
policy_arn = result.get('policy_arn')
assert status == 'ok'
assert policy_arn == expected_policy_arn
assert self.iam.policy_info (policy_arn).get('PolicyName' ) == 'test_policy'
assert self.iam.policy_details(policy_arn).get('policy_version').get('Document') == new_policy_document
assert self.iam.policy_delete(policy_arn) is True
def test_policy_name_exists(self):
assert self.iam.policy_exists_by_name('aaa' ) is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole' ) is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole','/service-role' ) is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole', '/service-role', 'aws') is True
def test_policy_info(self):
assert self.iam.policy_info('AAAAAA') is None
#def test_user_create(self): # convered in setUpClass
# self.iam.user_create()
#def test_user_delete(self): # convered in tearDownClass
# Dev.pprint(self.iam.user_delete())
def test_user_exists(self):
assert self.iam .user_exists() is True
assert self.iam.set_user_name('aAAA').user_exists() is False
def test_user_info(self):
user = self.iam.user_info()
(
Assert(user).field_is_equal('Arn' , test_user_arn)
.field_is_equal('Path' , '/')
.field_is_equal('UserName', test_user)
)
assert self.iam.set_user_name('AAAA').user_info().get('error') is not None
def test_users(self):
assert len(list(self.iam.users())) > 5
def test_role_create(self):
self.iam.role_delete()
role = self.iam.role_create(policy_document)
(
Assert(role).field_is_equal('Arn' ,test_role_arn)
.field_is_equal('Path' ,'/')
.field_is_equal('RoleName' , test_role)
.field_is_equal('AssumeRolePolicyDocument', policy_document)
)
assert self.iam.role_arn() == test_role_arn
def test_role_create_assume_role(self):
sts = STS()
current_user_arn = sts.caller_identity_arn()
original_policy = {'Statement': [ { 'Action' : 'sts:AssumeRole',
'Effect' : 'Allow',
'Principal': { 'Service': 'codebuild.amazonaws.com'}}]}
new_policy = {'Statement': [{'Action' : 'sts:AssumeRole',
'Effect' : 'Allow',
'Principal': {'AWS': current_user_arn } }]}
test_role = IAM(role_name="temp_role_to_test_assume_role")
test_role.role_create(original_policy)
role_arn = test_role.role_arn()
current_assume_policy = test_role.role_assume_policy()
test_role.role_assume_policy_update(new_policy)
for i in range(0,15):
with Catch(log_exception=False):
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
pprint('got credentials')
break
print(f'after {i} seconds')
wait(1)
assert sts.assume_role(role_arn=role_arn).get('Credentials') is not None
test_role.role_assume_policy_update(current_assume_policy)
assert test_role.role_assume_policy() == current_assume_policy
test_role.role_delete()
# credentials = sts.assume_role(role_arn=role_arn).get('Credentials')
# aws_access_key_id = credentials.get('AccessKeyId')
# aws_secret_access_key = credentials.get('SecretAccessKey')
# aws_session_token = credentials.get('SessionToken')
#
# import boto3
# session = boto3.Session(aws_access_key_id=aws_access_key_id,
# aws_secret_access_key=aws_secret_access_key,
# aws_session_token=aws_session_token)
def test_role_info(self):
role = self.iam.set_role_name(test_role).role_info() # also tests the set_role_name function
assert role.get('Arn' ) == test_role_arn
assert role.get('RoleName') == test_role
@pytest.mark.skip('Fix test')
def test_role_policies(self):
policies = self.iam.role_policies()
assert len(set(policies)) == 0
iam = IAM(role_name='AWSServiceRoleForAPIGateway')
assert iam.role_policies() == {'APIGatewayServiceRolePolicy': 'arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy'}
assert len(iam.role_policies_statements().get('APIGatewayServiceRolePolicy')[0].get('Action')) > 10
def test_role_policies_attach__role_policies_detach(self):
policy_name = 'test_policy'
policy_document = {"Version": "2012-10-17", # refactor this with policy helper
"Statement": [{ "Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"}]}
policy_arn = self.iam.policy_create(policy_name, policy_document).get('policy_arn')
assert len(list(self.iam.role_policies())) == 0
self.iam.role_policy_attach(policy_arn)
assert list(self.iam.role_policies()) == ['test_policy']
self.iam.role_policy_detach(policy_arn)
assert self.iam.policy_exists(policy_arn) is True
assert self.iam.policy_delete(policy_arn) is True # this will not delete a policy that is attached
assert self.iam.policy_exists(policy_arn) is False
#@pytest.mark.skip('Fix test')
def test_roles(self):
assert len(self.iam.roles()) > 5
def test_user_access_key_create__delete(self):
access_key = self.iam.user_access_key_create()
assert self.iam.access_key__wait_until_key_is_working (access_key,success_count=1) is True
self.iam.user_access_keys_delete_all()
assert self.iam.access_key__wait_until_key_is_not_working(access_key, success_count=1) is True
class CodeBuild:
def __init__(self, project_name, role_name):
self.codebuild = Session().client('codebuild')
self.iam = IAM(role_name=role_name)
self.project_name = project_name
return
def _invoke_via_paginator(self, method, field_id, use_paginator, **kwargs):
paginator = self.codebuild.get_paginator(method)
for page in paginator.paginate(**kwargs):
for id in page.get(field_id):
yield id
if use_paginator is False:
return
def all_builds_ids(self, use_paginator = False):
return self._invoke_via_paginator('list_builds','ids',use_paginator)
def build_info(self, build_id):
builds = self.codebuild.batch_get_builds(ids=[build_id]).get('builds')
return Misc.array_pop(builds,0)
def build_start(self):
kvargs = { 'projectName': self.project_name }
return self.codebuild.start_build(**kvargs).get('build').get('arn')
def build_wait_for_completion(self, build_id, sleep_for=0.5, max_attempts=20, log_status=False):
for i in range(0,max_attempts):
build_info = self.build_info(build_id)
build_status = build_info.get('buildStatus')
current_phase = build_info.get('currentPhase')
if log_status:
Dev.pprint("[{0}] {1} {2}".format(i,build_status,current_phase))
if build_status != 'IN_PROGRESS':
return build_info
sleep(sleep_for)
return None
def policies_create(self, policies): # does not update, only add new ones
policies_arns = []
role_policies = list(self.iam.role_policies().keys())
for base_name, policy in policies.items():
policy_name = "{0}_{1}".format(base_name, self.project_name)
if policy_name in role_policies:
continue
policies_arns.append(self.iam.policy_create(policy_name,policy).get('policy_arn'))
return policies_arns
def project_builds(self,ids):
return self.codebuild.batch_get_builds(ids=ids)
def project_create(self, project_repo, service_role):
kvargs = {
'name': self.project_name,
'source': {'type': 'GITHUB',
'location': project_repo},
'artifacts': {'type': 'NO_ARTIFACTS'},
'environment': {'type': 'LINUX_CONTAINER',
'image': 'aws/codebuild/python:3.7.1-1.7.0',
'computeType': 'BUILD_GENERAL1_SMALL'},
'serviceRole': service_role
}
return self.codebuild.create_project(**kvargs)
def project_delete(self):
if self.project_exists() is False: return False
self.codebuild.delete_project(name=self.project_name)
return self.project_exists() is False
def project_exists(self):
return self.project_name in self.projects()
def project_info(self):
projects = Misc.get_value(self.codebuild.batch_get_projects(names=[self.project_name]),'projects',[])
return Misc.array_pop(projects,0)
def project_builds_ids(self, project_name, use_paginator=False):
if use_paginator:
kwargs = { 'projectName' : project_name }
else:
kwargs = { 'projectName' : project_name ,
'sortOrder' : 'DESCENDING' }
return self._invoke_via_paginator('list_builds_for_project', 'ids',use_paginator, **kwargs)
def projects(self):
return self.codebuild.list_projects().get('projects')
class Test_IAM(TestCase):
@classmethod
def setUpClass(cls):
import warnings
warnings.filterwarnings("ignore",
category=ResourceWarning,
message="unclosed.*<ssl.SSLSocket.*>")
iam = IAM(user_name=test_user, role_name=test_role)
if iam.user_exists() is False:
iam.user_create()
if iam.role_exists() is False:
iam.role_create(policy_document)
@classmethod
def tearDownClass(cls):
if delete_created:
iam = IAM(user_name=test_user, role_name=test_role)
iam.user_delete()
assert iam.user_exists() is False
iam.role_delete()
assert iam.role_exists() is False
assert iam.role_arn() is None
def setUp(self):
import warnings
warnings.filterwarnings("ignore",
category=ResourceWarning,
message="unclosed.*<ssl.SSLSocket.*>")
self.iam = IAM(user_name=test_user, role_name=test_role)
@unittest.skip(
"Doesn't work in CodeBuild since there is only one configuration in there"
)
def test_account_id(self):
account_id_1 = self.iam.account_id('gs-detect-aws')
assert Globals.aws_session_profile_name == 'gs-detect-aws'
self.iam._account_id = None
self.iam._sts = None
account_id_2 = self.iam.account_id('default')
assert Globals.aws_session_profile_name == 'default'
assert account_id_1 != account_id_2
self.iam._account_id = None
self.iam._sts = None
account_id_3 = self.iam.account_id()
assert Globals.aws_session_profile_name == 'default'
assert account_id_2 == account_id_3
# def test_assume_role(self): # getting `(AccessDenied) when calling the AssumeRole operation: Access denied`
# role_arn = 'arn:aws:iam::244560807427:role/temp_role_for_lambda_invocation' # needs to be non account_id specific
# role_session_name = 'temp_role_for_test'
# Dev.pprint(self.iam.assume_role(role_arn,role_session_name))
def test_caller_identity(self):
assert set(self.iam.caller_identity()) == {'UserId', 'Account', 'Arn'}
def test_groups(self):
assert len(self.iam.groups()) > 5
def test_policies(self):
assert len(self.iam.policies()) > 500
def test_policy_arn(self):
#account_id = self.iam.account_id()
assert self.iam.policy_arn(
'aaa') == 'arn:aws:iam::{0}:policy/aaa'.format(account_id)
assert self.iam.policy_arn(
'aaa/bbb') == 'arn:aws:iam::{0}:policy/aaa/bbb'.format(account_id)
assert self.iam.policy_arn(
'aa', '/bb') == 'arn:aws:iam::{0}:policy/bb/aa'.format(account_id)
assert self.iam.policy_arn('aa', '/bb',
'cc') == 'arn:aws:iam::cc:policy/bb/aa'
assert self.iam.policy_arn(None) is None
self.iam.set_account_id('12345')
assert self.iam.policy_arn(
'aaa') == 'arn:aws:iam::{0}:policy/aaa'.format('12345')
def test_policy_create__policy_delete__policy_details(self):
policy_name = 'test_policy'
self.iam.policy_delete_by_name(policy_name)
new_policy_document = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"
}]
}
result = self.iam.policy_create(policy_name, new_policy_document)
expected_policy_arn = self.iam.policy_arn(policy_name)
status = result.get('status')
policy_arn = result.get('policy_arn')
assert status == 'ok'
assert policy_arn == expected_policy_arn
assert self.iam.policy_info(policy_arn).get(
'PolicyName') == 'test_policy'
assert self.iam.policy_details(policy_arn).get('policy_version').get(
'Document') == new_policy_document
assert self.iam.policy_delete(policy_arn) is True
def test_policy_name_exists(self):
assert self.iam.policy_exists_by_name('aaa') is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole') is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole',
'/service-role') is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole',
'/service-role', 'aws') is True
def test_policy_info(self):
assert self.iam.policy_info('AAAAAA') is None
#def test_user_create(self):
# self.iam.user_create()
#def test_user_delete(self):
# Dev.pprint(self.iam.user_delete())
def test_user_exists(self):
assert self.iam.user_exists() is True
assert self.iam.set_user_name('aAAA').user_exists() is False
def test_user_info(self):
user = self.iam.user_info()
(Assert(user).field_is_equal('Arn', test_user_arn).field_is_equal(
'Path', '/').field_is_equal('UserName', test_user))
assert self.iam.set_user_name('AAAA').user_info() is None
def test_users(self):
assert len(self.iam.users()) > 10
def test_role_create(self):
self.iam.role_delete()
role = self.iam.role_create(policy_document)
(Assert(role).field_is_equal('Arn', test_role_arn).field_is_equal(
'Path', '/').field_is_equal('RoleName', test_role).field_is_equal(
'AssumeRolePolicyDocument', policy_document))
assert self.iam.role_arn() == test_role_arn
def test_role_info(self):
role = self.iam.set_role_name(
test_role).role_info() # also tests the set_role_name function
assert role.get('Arn') == test_role_arn
assert role.get('RoleName') == test_role
def test_role_policies(self):
policies = self.iam.role_policies()
assert len(set(policies)) == 0
def test_role_policies(self):
iam = IAM(role_name='AWSBatchServiceRole')
assert iam.role_policies() == {
'AWSBatchServiceRole':
'arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole'
}
assert len(iam.role_policies_statements().get('AWSBatchServiceRole')
[0].get('Action')) == 59
def test_role_policies_attach__role_policies_detach(self):
policy_name = 'test_policy'
policy_document = {
"Version":
"2012-10-17", # refactor this with policy helper
"Statement": [{
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"
}]
}
policy_arn = self.iam.policy_create(policy_name,
policy_document).get('policy_arn')
assert len(list(self.iam.role_policies())) == 0
self.iam.role_policy_attach(policy_arn)
assert list(self.iam.role_policies()) == ['test_policy']
self.iam.role_policy_detach(policy_arn)
assert self.iam.policy_exists(policy_arn) is True
assert self.iam.policy_delete(
policy_arn
) is True # this will not delete a policy that is attached
assert self.iam.policy_exists(policy_arn) is False
def test_roles(self):
assert len(self.iam.roles()) > 70
