def test_role_policies(self):
policies = self.iam.role_policies()
assert len(set(policies)) == 0
iam = IAM(role_name='AWSServiceRoleForAPIGateway')
assert iam.role_policies() == {'APIGatewayServiceRolePolicy': 'arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy'}
assert len(iam.role_policies_statements().get('APIGatewayServiceRolePolicy')[0].get('Action')) > 10
def __init__(self, policy_name=None, policy_path=None):
self.iam = IAM()
self.policy_name = policy_name
self.version = "2012-10-17"
self.statements = []
self.policy_path = policy_path
self.account_id = self.iam.account_id()
def test_trail_create(self):
iam = IAM()
account_id = iam.account_id()
region = iam.region()
s3_key_prefix = self.trail_name
sns_topic_name = None
self.result = self.cloud_trail.trail_create(self.trail_name, account_id, region, self.s3_bucket, s3_key_prefix)
def test_role_policies(self):
iam = IAM(role_name='AWSBatchServiceRole')
assert iam.role_policies() == {
'AWSBatchServiceRole':
'arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole'
}
assert len(iam.role_policies_statements().get('AWSBatchServiceRole')
[0].get('Action')) == 59
def __init__(self, aws_user='******', region='eu-west-2'):
self.profile_name = '832789828058_AdministratorAccess' # this uses temporary tokens which expire after 12 hours (so this value will need to be reset before execution)
self.account_id = 'cloudsdkcustomera-glasswall'
self.region = region
self.osbot_setup = OSBot_Setup(profile_name=self.profile_name,
account_id=self.account_id,
region_name=self.region)
self.aws_user = aws_user
self.path_temp_credentials = f'/tmp/temp_credentials_{self.aws_user}.json'
self.iam = IAM(user_name=self.aws_user)
def tearDownClass(cls):
code_build = CodeBuild(project_name=project_name,role_name=project_name)
iam = IAM(role_name=project_name)
assert code_build.project_exists() is True
assert iam.role_exists() is True
if delete_on_teardown:
code_build.project_delete()
iam .role_delete()
assert code_build.project_exists() is False
assert iam.role_exists() is False
def create_test_for_lambda_invocation(self):
role_name = self.temp_aws_roles.role_name__for_lambda_invocation
role_arn = 'arn:aws:iam::{0}:role/{1}'.format(self.account_id,
role_name)
assert role_arn == self.temp_aws_roles.for_lambda_invocation(
delete_existing=True)
self.iam = IAM().set_role_name(role_name)
policies_statements = self.iam.role_policies_statements(
just_statements=True)
resource = policies_statements[0].get('Resource')[0]
assert self.account_id in resource # confirm account_id value is in there (regression test for bug)
class Temp_User:
def __init__(self):
self.user_name = f'temp_user_{random_string()}'
self.iam = IAM(user_name=self.user_name)
def __enter__(self):
self.iam.user_create()
return self
def __exit__(self, type, value, traceback):
self.iam.user_delete()
def test_user_privs(self):
user_name = 'AlekhAnejaUpwork'
access_key_id = 'AKIA3NUU5XSYZRQYXMP2'
iam_user = IAM(user_name=user_name)
assert iam_user.user_exists()
assert list_set(iam_user.user_access_keys(index_by='AccessKeyId')) == [
access_key_id
]
pprint(iam_user.user_polices())
def tearDownClass(cls):
if delete_created:
iam = IAM(user_name=test_user,role_name=test_role)
iam.user_delete()
assert iam.user_exists() is False
iam.role_delete()
assert iam.role_exists() is False
assert iam.role_arn() is None
def test_log_files__send_to_elk(self):
iam = IAM()
account_id = iam.account_id()
region = iam.region()
year = '2020'
month = '02'
day = '16'
max = 200
log_files = self.cloud_trail.log_files(self.trail_name, account_id, region, year, month, day)
records = self.cloud_trail.log_files_records(self.trail_name, log_files[0:max])
self.result = self.send_to_elk(records, 'eventID')
class IAM_Role:
def __init__(self, role_name):
self.role_name = role_name
self.iam = IAM(role_name=self.role_name)
self.role_arn = None
self.policy_arn = None
def add_policy_for__lambda(self):
temp_policy_name = 'policy_{0}'.format(self.role_name)
cloud_watch_arn = "arn:aws:logs:{0}:{1}:log-group:/aws/lambda/*".format(
'eu-west-2', '244560807427')
iam_policy = IAM_Policy(temp_policy_name)
self.policy_arn = iam_policy.add_cloud_watch(
cloud_watch_arn).create().get('policy_arn')
self.iam.role_policy_attach(self.policy_arn)
return self
def create_for__lambda(self):
result = self.create_for_service('lambda.amazonaws.com')
if result.get('status') == 'ok':
self.add_policy_for__lambda()
return result
def create_for__code_build(self):
return self.create_for_service('codebuild.amazonaws.com')
def create_for_service(self, service):
role_arn = self.iam.role_arn()
if role_arn:
return {
'status': 'warning',
'data': 'role already exists',
'role_name': self.iam.role_name,
'role_arn': role_arn
}
else:
policy_document = {
'Statement': [{
'Action': 'sts:AssumeRole',
'Effect': 'Allow',
'Principal': {
'Service': service
}
}]
}
data = self.iam.role_create(policy_document)
return {
'status': 'ok',
'data': data,
'role_name': self.iam.role_name,
'role_arn': data.get('Arn')
}
def integration_create__lambda(self, api_id, resource_id, lambda_name, http_method):
iam = IAM()
aws_acct_id = iam.account_id()
aws_region = iam.region()
input_type = 'AWS_PROXY'
uri = f'arn:aws:apigateway:{aws_region}:lambda:path/2015-03-31/functions/arn:aws:lambda:{aws_region}:{aws_acct_id}:function:{lambda_name}/invocations'
integration_http_method = 'POST'
try:
return self.api_gateway.put_integration(restApiId=api_id, resourceId=resource_id, httpMethod=http_method,
integrationHttpMethod=integration_http_method,type=input_type, uri=uri)
except Exception as error:
return f'{error}'
def integration_add_permission_to_lambda(self,api_id, lambda_name):
# create permission to allow lambda function to be invoked by API Gateway
iam = IAM()
aws_acct_id = iam.account_id()
aws_region = iam.region()
aws_lambda = Lambda(lambda_name)
function_arn = aws_lambda.function_Arn()#'gw_bot.lambdas.dev.hello_world'
statement_id = 'allow-api-gateway-invoke'
action = 'lambda:InvokeFunction'
principal = 'apigateway.amazonaws.com'
source_arn = f'arn:aws:execute-api:{aws_region}:{aws_acct_id}:{api_id}/*/GET/'
aws_lambda.permission_delete(function_arn, statement_id) # remove in case there was already a permission with this name
return aws_lambda.permission_add(function_arn, statement_id, action, principal, source_arn)
def policy_add_sqs_permissions_to_lambda_role(self, lambda_name):
aws_lambda = Lambda(lambda_name)
iam_role_name = aws_lambda.configuration().get('Role').split(
':role/').pop()
IAM(role_name=iam_role_name).role_policy_attach(
self.arn_aws_policy_service_sqs_lambda)
return iam_role_name
def setUpClass(cls):
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
iam = IAM(user_name=test_user, role_name=test_role)
if iam.user_exists() is False:
iam.user_create()
if iam.role_exists() is False:
iam.role_create(policy_document)
def _test_lambda_write_cloud_watch__with_asserts(self):
group_name = '/unit-tests/test_log_group'
stream_name = Misc.random_string_and_numbers(prefix='tmp_stream_')
message = 'this is a message sent from an lambda function'
lambda_name = 'osbot_aws.lambdas.dev.write_cloud_watch_log'
log_group_arn = 'arn:aws:logs:eu-west-2:244560807427:log-group:{0}*'.format(
group_name)
policy_name = 'temp_policy_for_lambda_write_cloud_watch'
role_name = 'temp_role_for_lambda_invocation'
policy_actions = ['logs:PutLogEvents']
logs = Logs(group_name=group_name, stream_name=stream_name)
logs.group_create()
logs.stream_create()
iam_role = IAM(role_name=role_name)
iam_policy = IAM_Policy(policy_name=policy_name)
iam_policy.add_statement_allow(policy_actions, [log_group_arn])
policy_arn = iam_policy.create(
delete_before_create=True).get('policy_arn')
assert iam_policy.exists() is True
assert iam_role.role_exists() is True
assert logs.group_exists() is True
assert logs.stream_exists() is True
assert set(iam_role.role_policies()) == {
'AWSXrayWriteOnlyAccess', 'policy_temp_role_for_lambda_invocation'
}
iam_role.role_policy_attach(policy_arn)
assert set(iam_role.role_policies()) == {
'AWSXrayWriteOnlyAccess', 'policy_temp_role_for_lambda_invocation',
'temp_policy_for_lambda_write_cloud_watch'
}
sleep(10) # wait for AWS to propagate role update
payload = {
'group_name': group_name,
'stream_name': stream_name,
'message': message
}
lambda_obj = Lambda_Package(lambda_name) #.update_with_root_folder()
result = lambda_obj.invoke(payload)
sleep(1) # wait for Cloudwatch to update
assert result.get('status') == 'ok'
assert logs.messages() == [message]
assert iam_policy.delete() is True
assert logs.group_delete() is True
assert logs.group_exists() is False
assert set(iam_role.role_policies()) == {
'AWSXrayWriteOnlyAccess', 'policy_temp_role_for_lambda_invocation'
}
def add_notification(source_arn):
result = lambda_obj._lambda.client().permission_add(
FunctionName=lambda_arn,
StatementId='1',
Action='lambda:InvokeFunction',
Principal='s3.amazonaws.com',
SourceArn=source_arn,
SourceAccount=IAM().account_id())
Dev.pprint(result)
def setUp(self):
super().setUp()
#Deploy().setup()
self.project_name = 'OSBot-Jupyter'
self.iam = IAM()
self.account_id = self.iam.account_id()
self.region = self.iam.region()
self.github_org = 'filetrust'
self.source_version = 'master'
self.build_spec = 'buildspec.yml'
self.docker_type = 'LINUX_CONTAINER'
#self.docker_image = '{0}.dkr.ecr.eu-west-1.amazonaws.com/osbot-jupyter:latest'.format(self.account_id),
self.compute_type = 'BUILD_GENERAL1_MEDIUM'
#self.api = Create_Code_Build(project_name=self.project_name)
self.api = Create_Code_Build(project_name=self.project_name,
github_org=self.github_org,
source_version=self.source_version,
docker_type=self.docker_type,
compute_type=self.compute_type,
build_spec=self.build_spec)
def stream_create(self, stream_name, s3_bucket, s3_prefix, compression='UNCOMPRESSED'):
account_id = IAM().account_id()
role = 'firehose_delivery_role'
params = { "DeliveryStreamName": stream_name,
"DeliveryStreamType": "DirectPut",
'ExtendedS3DestinationConfiguration': {
"RoleARN" : f"arn:aws:iam::{account_id}:role/{role}",
"BucketARN" : f"arn:aws:s3:::{s3_bucket}",
"Prefix" : f"{s3_prefix}/",
"CompressionFormat": compression }}
return self.firehose().create_delivery_stream(**params)
def delete_cluster_task_definition_tasks_roles(self):
# stop task
self.stop()
self.wait_for_task_stopped()
# delete task_definitions
task_definitions_arns = self.ecs.task_definitions_arns(task_family=self.task_family)
for task_definition_arn in task_definitions_arns:
self.ecs.task_definition_delete(task_definition_arn=task_definition_arn)
# delete roles
task_definition_config = self.ecs.task_definition_setup(task_family=self.task_family, image_name=self.image_name)
execution_role_name = task_definition_config.get('execution_role_name')
task_role_name = task_definition_config.get('task_role_name')
IAM(role_name=execution_role_name).role_delete()
IAM(role_name=task_role_name ).role_delete()
# delete cluster
self.ecs.cluster_delete(cluster_arn=self.cluster_arn())
def policy_create_for_execution_role(self, role_name):
region = 'eu-west-2'
cloud_watch_arn = "arn:aws:logs:{0}:{1}:log-group:awslogs-*".format(
region, self.account_id)
role_policy = {
"Version":
"2008-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
"Action": "sts:AssumeRole"
}]
}
policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer", "ecr:GetRepositoryPolicy",
"ecr:DescribeRepositories", "ecr:ListImages",
"ecr:DescribeImages", "ecr:BatchGetImage"
],
"Resource":
"*"
}, {
"Effect":
"Allow",
"Action": ["logs:CreateLogStream", "logs:PutLogEvents"],
"Resource": [cloud_watch_arn]
}]
}
policy_name = 'policy_for_{0}'.format(role_name)
iam = IAM(role_name=role_name)
iam.role_create(role_policy)
iam.policy_delete(policy_name)
policy_arn = iam.policy_create(policy_name, policy).get('policy_arn')
iam.role_policy_attach(policy_arn)
def test_get_available_log_files(self):
iam = IAM()
account_id = iam.account_id()
region = iam.region()
log_type = 'CloudTrail'
year = '2020'
month = '02'
day = '16'
hour = ''
minute = ''
s3_prefix = f'{self.s3_key_prefix}/AWSLogs/{account_id}/' \
f'{log_type}/{region}/{year}/{month}/{day}/'
s3 = self.cloud_trail.s3
s3_files = s3.find_files(self.s3_bucket, prefix=s3_prefix)
print('-------')
total = 0
for s3_file in sorted(s3_files):
contents = s3.file_contents_from_gzip(self.s3_bucket,s3_file)
records = json.loads(contents).get("Records")
total += len(records)
print(f'{len(records): 4} : {total : 4} : {s3_file.split("/").pop()}')
def policy_create_for_task_role(self, role_name):
policy = {
"Version":
"2008-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
"Action": "sts:AssumeRole"
}]
}
IAM(role_name=role_name).role_create(policy)
def setUpClass(cls):
code_build = CodeBuild(project_name=project_name, role_name=project_name)
iam = IAM(role_name=project_name)
if delete_on_setup:
code_build.project_delete()
iam.role_delete()
if code_build.project_exists() is False:
assert code_build.project_exists() is False
iam.role_create(assume_policy) # create role
assert iam.role_info().get('Arn') == service_role # confirm the role exists
sleep(1)
code_build.project_create(project_repo, service_role) # in a non-deterministic way, this sometimes throws the error: CodeBuild is not authorized to perform: sts:AssumeRole
class IAM_Policy:
def __init__(self, policy_name=None, policy_path=None):
self.iam = IAM()
self.policy_name = policy_name
self.version = "2012-10-17"
self.statements = []
self.policy_path = policy_path
self.account_id = self.iam.account_id()
def add_cloud_watch(self, resource_arn):
return self.add_statement_allow([
"logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"
], [resource_arn])
def add_statement(self, effect, actions, resources):
self.statements.append({
"Effect": effect,
"Action": actions,
"Resource": resources
})
return self
def add_statement_allow(self, actions, resources):
return self.add_statement('Allow', actions, resources)
def create(self, delete_before_create=False):
if self.policy_name is None:
return {'status': 'error', 'data': 'policy name is None'}
return self.iam.policy_create(
self.policy_name,
self.statement(),
delete_before_create=delete_before_create)
def delete(self):
return self.iam.policy_delete(self.policy_arn())
def exists(self):
return self.iam.policy_exists(self.policy_arn())
def policy_arn(self):
return self.iam.policy_arn(self.policy_name, self.policy_path,
self.account_id)
def statement(self):
return {'Version': self.version, 'Statement': self.statements}
def statement_from_aws(self):
return self.iam.policy_statement(self.policy_arn())
def test_delete_cluster_task_definition_tasks_roles(self):
cluster_name = self.cluster_name
cluster_arn = self.cluster_arn
task_definition_arn = self.task_definition_arn
task__arn = self.fargate_task.task_arn
task_family = self.fargate_task.task_family
image_name = self.fargate_task.image_name
task_definition_config = self.fargate_task.ecs.task_definition_setup(
task_family=task_family, image_name=image_name)
task_role_name = task_definition_config.get('task_role_name')
execution_role_name = task_definition_config.get('execution_role_name')
iam_role_task = IAM(role_name=task_role_name)
iam_role_execution = IAM(role_name=execution_role_name)
# check that everything exists before
assert self.fargate_task.ecs.cluster_exists(
cluster_arn=cluster_arn) is True
assert self.fargate_task.ecs.task_definition_exists(
task_definition_arn=task_definition_arn) is True
assert self.fargate_task.ecs.task_exists(cluster_name=cluster_name,
task_arn=task__arn) is True
#assert self.fargate_task.status () == 'PROVISIONING' (is STOPPED when called from Teardown)
assert iam_role_task.role_exists()
assert iam_role_execution.role_exists()
self.fargate_task.delete_cluster_task_definition_tasks_roles()
# check that everything is deleted after
assert self.fargate_task.ecs.cluster_exists(
cluster_arn=cluster_arn) is False
assert self.fargate_task.ecs.task_definition_exists(
task_definition_arn=task_definition_arn) is False
assert self.fargate_task.status() == 'STOPPED'
assert iam_role_task.role_not_exists()
assert iam_role_execution.role_not_exists()
class test_Temp_Aws_Roles(TestCase):
def setUp(self):
self.temp_aws_roles = Temp_Aws_Roles()
self.account_id = '244560807427'
def test_for_lambda_invocation(self):
role_arn = self.temp_aws_roles.for_lambda_invocation()
assert role_arn == 'arn:aws:iam::244560807427:role/temp_role_for_lambda_invocation'
@unittest.skip(
'causes side effect with other tests, due to the fact that AWS takes a while to sync the privileges internally'
)
def create_test_for_lambda_invocation(self):
role_name = self.temp_aws_roles.role_name__for_lambda_invocation
role_arn = 'arn:aws:iam::{0}:role/{1}'.format(self.account_id,
role_name)
assert role_arn == self.temp_aws_roles.for_lambda_invocation(
delete_existing=True)
self.iam = IAM().set_role_name(role_name)
policies_statements = self.iam.role_policies_statements(
just_statements=True)
resource = policies_statements[0].get('Resource')[0]
assert self.account_id in resource # confirm account_id value is in there (regression test for bug)
def test_task_definition_create(self):
task_family = self.task_family
image_name = 'an_docker_image'
task_definition_config = self.ecs.task_definition_setup(
task_family=task_family, image_name=image_name)
log_group_name = task_definition_config.get('log_group_name')
log_stream_name = task_definition_config.get('log_stream_name')
task_definition = self.ecs.task_definition_create(
task_definition_config=task_definition_config)
task_definition_arn = task_definition.get('taskDefinitionArn')
revision = task_definition.get('revision')
assert task_definition_arn == self.ecs.task_definition_arn(
task_family, revision)
assert task_definition.get('family') == task_family
assert task_definition.get('cpu') == '256' # minimum value
assert task_definition.get('memory') == '256'
assert task_definition_arn in self.ecs.task_definitions(
task_family=self.task_family, index_by='taskDefinitionArn')
assert self.logs.log_group_exists(log_group_name) is True
assert self.logs.log_stream_exists(log_group_name,
log_stream_name) is True
assert self.ecs.task_definition_exists(task_definition_arn) is True
assert self.ecs.task_definition_delete(task_definition_arn) is True
assert self.ecs.task_definition_exists(task_definition_arn) is False
iam_task_role = IAM(
role_name=task_definition_config.get('task_role_name'))
iam_execution_role = IAM(
role_name=task_definition_config.get('execution_role_name'))
assert iam_task_role.role_delete() is True
assert iam_execution_role.role_delete() is True
def setUp(self):
super().setUp()
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>")
self.iam = IAM(user_name=test_user,role_name=test_role )