def test_trail_create(self):
iam = IAM()
account_id = iam.account_id()
region = iam.region()
s3_key_prefix = self.trail_name
sns_topic_name = None
self.result = self.cloud_trail.trail_create(self.trail_name, account_id, region, self.s3_bucket, s3_key_prefix)
def test_log_files__send_to_elk(self):
iam = IAM()
account_id = iam.account_id()
region = iam.region()
year = '2020'
month = '02'
day = '16'
max = 200
log_files = self.cloud_trail.log_files(self.trail_name, account_id, region, year, month, day)
records = self.cloud_trail.log_files_records(self.trail_name, log_files[0:max])
self.result = self.send_to_elk(records, 'eventID')
def integration_create__lambda(self, api_id, resource_id, lambda_name, http_method):
iam = IAM()
aws_acct_id = iam.account_id()
aws_region = iam.region()
input_type = 'AWS_PROXY'
uri = f'arn:aws:apigateway:{aws_region}:lambda:path/2015-03-31/functions/arn:aws:lambda:{aws_region}:{aws_acct_id}:function:{lambda_name}/invocations'
integration_http_method = 'POST'
try:
return self.api_gateway.put_integration(restApiId=api_id, resourceId=resource_id, httpMethod=http_method,
integrationHttpMethod=integration_http_method,type=input_type, uri=uri)
except Exception as error:
return f'{error}'
def integration_add_permission_to_lambda(self,api_id, lambda_name):
# create permission to allow lambda function to be invoked by API Gateway
iam = IAM()
aws_acct_id = iam.account_id()
aws_region = iam.region()
aws_lambda = Lambda(lambda_name)
function_arn = aws_lambda.function_Arn()#'gw_bot.lambdas.dev.hello_world'
statement_id = 'allow-api-gateway-invoke'
action = 'lambda:InvokeFunction'
principal = 'apigateway.amazonaws.com'
source_arn = f'arn:aws:execute-api:{aws_region}:{aws_acct_id}:{api_id}/*/GET/'
aws_lambda.permission_delete(function_arn, statement_id) # remove in case there was already a permission with this name
return aws_lambda.permission_add(function_arn, statement_id, action, principal, source_arn)
class IAM_Policy:
def __init__(self, policy_name=None, policy_path=None):
self.iam = IAM()
self.policy_name = policy_name
self.version = "2012-10-17"
self.statements = []
self.policy_path = policy_path
self.account_id = self.iam.account_id()
def add_cloud_watch(self, resource_arn):
return self.add_statement_allow([
"logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"
], [resource_arn])
def add_statement(self, effect, actions, resources):
self.statements.append({
"Effect": effect,
"Action": actions,
"Resource": resources
})
return self
def add_statement_allow(self, actions, resources):
return self.add_statement('Allow', actions, resources)
def create(self, delete_before_create=False):
if self.policy_name is None:
return {'status': 'error', 'data': 'policy name is None'}
return self.iam.policy_create(
self.policy_name,
self.statement(),
delete_before_create=delete_before_create)
def delete(self):
return self.iam.policy_delete(self.policy_arn())
def exists(self):
return self.iam.policy_exists(self.policy_arn())
def policy_arn(self):
return self.iam.policy_arn(self.policy_name, self.policy_path,
self.account_id)
def statement(self):
return {'Version': self.version, 'Statement': self.statements}
def statement_from_aws(self):
return self.iam.policy_statement(self.policy_arn())
def test_get_available_log_files(self):
iam = IAM()
account_id = iam.account_id()
region = iam.region()
log_type = 'CloudTrail'
year = '2020'
month = '02'
day = '16'
hour = ''
minute = ''
s3_prefix = f'{self.s3_key_prefix}/AWSLogs/{account_id}/' \
f'{log_type}/{region}/{year}/{month}/{day}/'
s3 = self.cloud_trail.s3
s3_files = s3.find_files(self.s3_bucket, prefix=s3_prefix)
print('-------')
total = 0
for s3_file in sorted(s3_files):
contents = s3.file_contents_from_gzip(self.s3_bucket,s3_file)
records = json.loads(contents).get("Records")
total += len(records)
print(f'{len(records): 4} : {total : 4} : {s3_file.split("/").pop()}')
class Test_IAM(Test_Helper):
@classmethod
def setUpClass(cls):
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
iam = IAM(user_name=test_user, role_name=test_role)
if iam.user_exists() is False:
iam.user_create()
if iam.role_exists() is False:
iam.role_create(policy_document)
@classmethod
def tearDownClass(cls):
if delete_created:
iam = IAM(user_name=test_user,role_name=test_role)
iam.user_delete()
assert iam.user_exists() is False
iam.role_delete()
assert iam.role_exists() is False
assert iam.role_arn() is None
def setUp(self):
super().setUp()
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
self.iam = IAM(user_name=test_user,role_name=test_role )
# ------ tests ------
@unittest.skip("Doesn't work in CodeBuild since there is only one configuration in there")
def test_account_id(self):
account_id_1 = self.iam.account_id('gs-detect-aws') # todo: rewrite since account_id doesn't take this parameter any more
assert AWS_Config().aws_session_profile_name() == 'gs-detect-aws'
self.iam._account_id = None
self.iam._sts = None
account_id_2 = self.iam.account_id('default')
assert AWS_Config().aws_session_profile_name() == 'default'
assert account_id_1 != account_id_2
self.iam._account_id = None
self.iam._sts = None
account_id_3 = self.iam.account_id()
assert AWS_Config().aws_session_profile_name() == 'default'
assert account_id_2 == account_id_3
@pytest.mark.skip('Fix test')
def test_access_keys(self):
assert len(self.iam.access_keys(index_by='AccessKeyId')) > 0
assert len(self.iam.access_keys(group_by='UserName' )) > 0
def test_caller_identity(self):
assert set(self.iam.caller_identity()) == {'UserId', 'Account', 'Arn'}
@pytest.mark.skip('Fix test')
def test_groups(self):
assert len(self.iam.groups()) > 5
def test_policies(self):
assert len(self.iam.policies()) > 500
def test_policy_arn(self):
assert self.iam.policy_arn('aaa' ) == 'arn:aws:iam::{0}:policy/aaa' .format(account_id)
assert self.iam.policy_arn('aaa/bbb' ) == 'arn:aws:iam::{0}:policy/aaa/bbb'.format(account_id)
assert self.iam.policy_arn('aa','/bb' ) == 'arn:aws:iam::{0}:policy/bb/aa' .format(account_id)
assert self.iam.policy_arn('aa','/bb','cc') == 'arn:aws:iam::cc:policy/bb/aa'
assert self.iam.policy_arn(None) is None
def test_policy_create__policy_delete__policy_details(self):
policy_name = 'test_policy'
self.iam.policy_delete_by_name(policy_name)
new_policy_document = { "Version": "2012-10-17",
"Statement": [ { "Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*" }]}
result = self.iam.policy_create(policy_name, new_policy_document)
expected_policy_arn = self.iam.policy_arn(policy_name)
status = result.get('status')
policy_arn = result.get('policy_arn')
assert status == 'ok'
assert policy_arn == expected_policy_arn
assert self.iam.policy_info (policy_arn).get('PolicyName' ) == 'test_policy'
assert self.iam.policy_details(policy_arn).get('policy_version').get('Document') == new_policy_document
assert self.iam.policy_delete(policy_arn) is True
def test_policy_name_exists(self):
assert self.iam.policy_exists_by_name('aaa' ) is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole' ) is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole','/service-role' ) is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole', '/service-role', 'aws') is True
def test_policy_info(self):
assert self.iam.policy_info('AAAAAA') is None
#def test_user_create(self): # convered in setUpClass
# self.iam.user_create()
#def test_user_delete(self): # convered in tearDownClass
# Dev.pprint(self.iam.user_delete())
def test_user_exists(self):
assert self.iam .user_exists() is True
assert self.iam.set_user_name('aAAA').user_exists() is False
def test_user_info(self):
user = self.iam.user_info()
(
Assert(user).field_is_equal('Arn' , test_user_arn)
.field_is_equal('Path' , '/')
.field_is_equal('UserName', test_user)
)
assert self.iam.set_user_name('AAAA').user_info().get('error') is not None
def test_users(self):
assert len(list(self.iam.users())) > 5
def test_role_create(self):
self.iam.role_delete()
role = self.iam.role_create(policy_document)
(
Assert(role).field_is_equal('Arn' ,test_role_arn)
.field_is_equal('Path' ,'/')
.field_is_equal('RoleName' , test_role)
.field_is_equal('AssumeRolePolicyDocument', policy_document)
)
assert self.iam.role_arn() == test_role_arn
def test_role_create_assume_role(self):
sts = STS()
current_user_arn = sts.caller_identity_arn()
original_policy = {'Statement': [ { 'Action' : 'sts:AssumeRole',
'Effect' : 'Allow',
'Principal': { 'Service': 'codebuild.amazonaws.com'}}]}
new_policy = {'Statement': [{'Action' : 'sts:AssumeRole',
'Effect' : 'Allow',
'Principal': {'AWS': current_user_arn } }]}
test_role = IAM(role_name="temp_role_to_test_assume_role")
test_role.role_create(original_policy)
role_arn = test_role.role_arn()
current_assume_policy = test_role.role_assume_policy()
test_role.role_assume_policy_update(new_policy)
for i in range(0,15):
with Catch(log_exception=False):
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
sts.assume_role(role_arn=role_arn)
pprint('got credentials')
break
print(f'after {i} seconds')
wait(1)
assert sts.assume_role(role_arn=role_arn).get('Credentials') is not None
test_role.role_assume_policy_update(current_assume_policy)
assert test_role.role_assume_policy() == current_assume_policy
test_role.role_delete()
# credentials = sts.assume_role(role_arn=role_arn).get('Credentials')
# aws_access_key_id = credentials.get('AccessKeyId')
# aws_secret_access_key = credentials.get('SecretAccessKey')
# aws_session_token = credentials.get('SessionToken')
#
# import boto3
# session = boto3.Session(aws_access_key_id=aws_access_key_id,
# aws_secret_access_key=aws_secret_access_key,
# aws_session_token=aws_session_token)
def test_role_info(self):
role = self.iam.set_role_name(test_role).role_info() # also tests the set_role_name function
assert role.get('Arn' ) == test_role_arn
assert role.get('RoleName') == test_role
@pytest.mark.skip('Fix test')
def test_role_policies(self):
policies = self.iam.role_policies()
assert len(set(policies)) == 0
iam = IAM(role_name='AWSServiceRoleForAPIGateway')
assert iam.role_policies() == {'APIGatewayServiceRolePolicy': 'arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy'}
assert len(iam.role_policies_statements().get('APIGatewayServiceRolePolicy')[0].get('Action')) > 10
def test_role_policies_attach__role_policies_detach(self):
policy_name = 'test_policy'
policy_document = {"Version": "2012-10-17", # refactor this with policy helper
"Statement": [{ "Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"}]}
policy_arn = self.iam.policy_create(policy_name, policy_document).get('policy_arn')
assert len(list(self.iam.role_policies())) == 0
self.iam.role_policy_attach(policy_arn)
assert list(self.iam.role_policies()) == ['test_policy']
self.iam.role_policy_detach(policy_arn)
assert self.iam.policy_exists(policy_arn) is True
assert self.iam.policy_delete(policy_arn) is True # this will not delete a policy that is attached
assert self.iam.policy_exists(policy_arn) is False
#@pytest.mark.skip('Fix test')
def test_roles(self):
assert len(self.iam.roles()) > 5
def test_user_access_key_create__delete(self):
access_key = self.iam.user_access_key_create()
assert self.iam.access_key__wait_until_key_is_working (access_key,success_count=1) is True
self.iam.user_access_keys_delete_all()
assert self.iam.access_key__wait_until_key_is_not_working(access_key, success_count=1) is True
class test_OSBot_Jupyter_Create_Code_Build(Test_Helper):
def setUp(self):
super().setUp()
#Deploy().setup()
self.project_name = 'OSBot-Jupyter'
self.iam = IAM()
self.account_id = self.iam.account_id()
self.region = self.iam.region()
self.github_org = 'filetrust'
self.source_version = 'master'
self.build_spec = 'buildspec.yml'
self.docker_type = 'LINUX_CONTAINER'
#self.docker_image = '{0}.dkr.ecr.eu-west-1.amazonaws.com/osbot-jupyter:latest'.format(self.account_id),
self.compute_type = 'BUILD_GENERAL1_MEDIUM'
#self.api = Create_Code_Build(project_name=self.project_name)
self.api = Create_Code_Build(project_name=self.project_name,
github_org=self.github_org,
source_version=self.source_version,
docker_type=self.docker_type,
compute_type=self.compute_type,
build_spec=self.build_spec)
#def create_project_with_container__osbot_jupyter(self):
# kvargs = {
# 'name' : self.api.project_name,
# 'source' : {'type' : 'GITHUB',
# 'location' : self.api.project_repo },
# 'artifacts' : {'type' : 'NO_ARTIFACTS' },
# 'environment' : {'type' : 'LINUX_CONTAINER' ,
# 'image' : '{0}.dkr.ecr.eu-west-1.amazonaws.com/osbot-jupyter:latest'.format(self.account_id) ,
# 'computeType' : 'BUILD_GENERAL1_SMALL' ,
# 'imagePullCredentialsType' : 'SERVICE_ROLE' },
# 'serviceRole' : self.api.service_role
# }
# return self.api.code_build.codebuild.create_project(**kvargs)
# this only needs to run once
def test_create_policies(self):
policies = self.api.policies__with_ecr_and_3_secrets()
self.api.create_role_and_policies(policies)
def test_create_code_build_and_trigger_first_build(self):
self.api.code_build.project_delete()
self.api.create_project_with_container__gs_docker_codebuild()
#self.create_project_with_container__osbot_jupyter()
#self.api.code_build.build_start()
def test_get_task_details(self):
from osbot_aws.apis.Logs import Logs
def find_starts(array, text):
return [item for item in array if item.startswith(text)]
def find_in(array, text):
return [item for item in array if text in item]
#build_id = 'OSBot-Jupyter:a553dda5-953a-41b8-ae91-e068cba4f56b'
result = self.api.code_build.project_builds_ids(self.api.project_name)
build_id = result.__next__() # get last one
build_info = self.api.code_build.build_info(build_id)
group_name = build_info.get('logs').get('groupName')
stream_name = build_info.get('logs').get('streamName')
#Dev.pprint(group_name,stream_name)
logs = Logs(group_name=group_name, stream_name=stream_name)
messages = logs.messages()
#ngrok_messages = find_starts(messages,'t=')
ngrok_url = find_in(
messages, 'name=command_line addr')[0].split('url=')[1].strip()
jupyter_token = find_in(messages,
'token=')[0].split('token=')[1].strip()
Dev.pprint("{0}?token={1}".format(ngrok_url, jupyter_token))
class CI_Setup:
def __init__(self, aws_user='******', region='eu-west-2'):
self.profile_name = '832789828058_AdministratorAccess' # this uses temporary tokens which expire after 12 hours (so this value will need to be reset before execution)
self.account_id = 'cloudsdkcustomera-glasswall'
self.region = region
self.osbot_setup = OSBot_Setup(profile_name=self.profile_name,
account_id=self.account_id,
region_name=self.region)
self.aws_user = aws_user
self.path_temp_credentials = f'/tmp/temp_credentials_{self.aws_user}.json'
self.iam = IAM(user_name=self.aws_user)
def check_profile_account_security_token_status(self):
"""
if this fails the current values set in the aws config need to be refreshed
:return: True if credentials are ok
"""
try:
self.iam.account_id()
return True
except ClientError as error:
assert error.response['Error'][
'Message'] == 'The security token included in the request is expired'
return False
@catch
def check_access_key_for_user(self):
(aws_access_key_id,
aws_secret_access_key) = self.get_access_key_for_user(new_key=False)
#return (aws_access_key, aws_secret)
s3 = S3()
return s3.buckets()
def create_aws_user_for_github(self):
role_name = f'role_for_{self.aws_user}'
#policy_name = f'policy_for_{user_name}'
#IAM(role_name=role_name)
#policy = IAM_Policy(policy_name=policy_name)
self.iam.user_create()
#iam.role_create(policy.statement())
assert self.iam.user_exists()
#assert iam.role_exists()
return
def get_access_key_for_user(self, new_key=True, delete_keys=True):
"""
get AWS access key for current user
:param new_key: when True (default) a new key will be created everytime this method is called
:param delete_keys: when True (default) all previous access keys will be deleted
:return: return tuple with (AccessKeyId,SecretAccessKey)
"""
if new_key or file_not_exists(
self.path_temp_credentials
): # only create key if file doesn't exist or new_key is False
if delete_keys: # by default make sure that there is only one valid key available
self.iam.user_access_keys_delete_all(
) # this will delete all current keys for this users
access_key = self.iam.user_access_key_create(
) # create new access keys
del access_key[
'CreateDate'] # because: Object of type datetime is not JSON serializable
json_save(self.path_temp_credentials,
access_key) # save them in temp file
else:
access_key = json_load(
self.path_temp_credentials) # load keys from temp file
return access_key.get('AccessKeyId'), access_key.get(
'SecretAccessKey'
) # return tuple with (access_key and secret_access_key)
class Test_IAM(TestCase):
@classmethod
def setUpClass(cls):
import warnings
warnings.filterwarnings("ignore",
category=ResourceWarning,
message="unclosed.*<ssl.SSLSocket.*>")
iam = IAM(user_name=test_user, role_name=test_role)
if iam.user_exists() is False:
iam.user_create()
if iam.role_exists() is False:
iam.role_create(policy_document)
@classmethod
def tearDownClass(cls):
if delete_created:
iam = IAM(user_name=test_user, role_name=test_role)
iam.user_delete()
assert iam.user_exists() is False
iam.role_delete()
assert iam.role_exists() is False
assert iam.role_arn() is None
def setUp(self):
import warnings
warnings.filterwarnings("ignore",
category=ResourceWarning,
message="unclosed.*<ssl.SSLSocket.*>")
self.iam = IAM(user_name=test_user, role_name=test_role)
@unittest.skip(
"Doesn't work in CodeBuild since there is only one configuration in there"
)
def test_account_id(self):
account_id_1 = self.iam.account_id('gs-detect-aws')
assert Globals.aws_session_profile_name == 'gs-detect-aws'
self.iam._account_id = None
self.iam._sts = None
account_id_2 = self.iam.account_id('default')
assert Globals.aws_session_profile_name == 'default'
assert account_id_1 != account_id_2
self.iam._account_id = None
self.iam._sts = None
account_id_3 = self.iam.account_id()
assert Globals.aws_session_profile_name == 'default'
assert account_id_2 == account_id_3
# def test_assume_role(self): # getting `(AccessDenied) when calling the AssumeRole operation: Access denied`
# role_arn = 'arn:aws:iam::244560807427:role/temp_role_for_lambda_invocation' # needs to be non account_id specific
# role_session_name = 'temp_role_for_test'
# Dev.pprint(self.iam.assume_role(role_arn,role_session_name))
def test_caller_identity(self):
assert set(self.iam.caller_identity()) == {'UserId', 'Account', 'Arn'}
def test_groups(self):
assert len(self.iam.groups()) > 5
def test_policies(self):
assert len(self.iam.policies()) > 500
def test_policy_arn(self):
#account_id = self.iam.account_id()
assert self.iam.policy_arn(
'aaa') == 'arn:aws:iam::{0}:policy/aaa'.format(account_id)
assert self.iam.policy_arn(
'aaa/bbb') == 'arn:aws:iam::{0}:policy/aaa/bbb'.format(account_id)
assert self.iam.policy_arn(
'aa', '/bb') == 'arn:aws:iam::{0}:policy/bb/aa'.format(account_id)
assert self.iam.policy_arn('aa', '/bb',
'cc') == 'arn:aws:iam::cc:policy/bb/aa'
assert self.iam.policy_arn(None) is None
self.iam.set_account_id('12345')
assert self.iam.policy_arn(
'aaa') == 'arn:aws:iam::{0}:policy/aaa'.format('12345')
def test_policy_create__policy_delete__policy_details(self):
policy_name = 'test_policy'
self.iam.policy_delete_by_name(policy_name)
new_policy_document = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"
}]
}
result = self.iam.policy_create(policy_name, new_policy_document)
expected_policy_arn = self.iam.policy_arn(policy_name)
status = result.get('status')
policy_arn = result.get('policy_arn')
assert status == 'ok'
assert policy_arn == expected_policy_arn
assert self.iam.policy_info(policy_arn).get(
'PolicyName') == 'test_policy'
assert self.iam.policy_details(policy_arn).get('policy_version').get(
'Document') == new_policy_document
assert self.iam.policy_delete(policy_arn) is True
def test_policy_name_exists(self):
assert self.iam.policy_exists_by_name('aaa') is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole') is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole',
'/service-role') is False
assert self.iam.policy_exists_by_name('AWSBatchServiceRole',
'/service-role', 'aws') is True
def test_policy_info(self):
assert self.iam.policy_info('AAAAAA') is None
#def test_user_create(self):
# self.iam.user_create()
#def test_user_delete(self):
# Dev.pprint(self.iam.user_delete())
def test_user_exists(self):
assert self.iam.user_exists() is True
assert self.iam.set_user_name('aAAA').user_exists() is False
def test_user_info(self):
user = self.iam.user_info()
(Assert(user).field_is_equal('Arn', test_user_arn).field_is_equal(
'Path', '/').field_is_equal('UserName', test_user))
assert self.iam.set_user_name('AAAA').user_info() is None
def test_users(self):
assert len(self.iam.users()) > 10
def test_role_create(self):
self.iam.role_delete()
role = self.iam.role_create(policy_document)
(Assert(role).field_is_equal('Arn', test_role_arn).field_is_equal(
'Path', '/').field_is_equal('RoleName', test_role).field_is_equal(
'AssumeRolePolicyDocument', policy_document))
assert self.iam.role_arn() == test_role_arn
def test_role_info(self):
role = self.iam.set_role_name(
test_role).role_info() # also tests the set_role_name function
assert role.get('Arn') == test_role_arn
assert role.get('RoleName') == test_role
def test_role_policies(self):
policies = self.iam.role_policies()
assert len(set(policies)) == 0
def test_role_policies(self):
iam = IAM(role_name='AWSBatchServiceRole')
assert iam.role_policies() == {
'AWSBatchServiceRole':
'arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole'
}
assert len(iam.role_policies_statements().get('AWSBatchServiceRole')
[0].get('Action')) == 59
def test_role_policies_attach__role_policies_detach(self):
policy_name = 'test_policy'
policy_document = {
"Version":
"2012-10-17", # refactor this with policy helper
"Statement": [{
"Effect": "Allow",
"Action": "lambda:InvokeFunction",
"Resource": "arn:aws:lambda:*:*:function:*"
}]
}
policy_arn = self.iam.policy_create(policy_name,
policy_document).get('policy_arn')
assert len(list(self.iam.role_policies())) == 0
self.iam.role_policy_attach(policy_arn)
assert list(self.iam.role_policies()) == ['test_policy']
self.iam.role_policy_detach(policy_arn)
assert self.iam.policy_exists(policy_arn) is True
assert self.iam.policy_delete(
policy_arn
) is True # this will not delete a policy that is attached
assert self.iam.policy_exists(policy_arn) is False
def test_roles(self):
assert len(self.iam.roles()) > 70