class test_Create_Code_Build(TestCase):
def setUp(self):
self.project_name = 'OSBot-jira'
self.api = Create_Code_Build(project_name=self.project_name)
def test_create(self):
policies = self.api.policies__with_ecr_and_3_secrets()
self.api.create_role_and_policies(policies)
sleep(5) # to give time for AWS to sync up internally
self.api.create_project_with_container__gs_docker_codebuild()
self.api.code_build.build_start()
class test_Create_Code_Build(TestCase):
def setUp(self):
self.project_name = 'pbx-gs-python-utils'
self.account_id = IAM().account_id()
self.api = Create_Code_Build(account_id=self.account_id,
project_name=self.project_name)
def test_create(self):
policies = self.api.policies__with_ecr_and_3_secrets()
self.api.create_role_and_policies(policies)
sleep(5) # to give time for AWS to sync up internally
self.api.create_project_with_container__gs_docker_codebuild()
self.api.code_build.build_start()
class test_Create_Code_Build(TestCase):
def setUp(self):
self.project_name = 'entityresolver'
self.profile_name = 'gs-detect-aws'
self.account_id = IAM().account_id(self.profile_name)
self.api = Create_Code_Build(account_id=self.account_id, project_name=self.project_name)
def test_create(self):
policies = self.api.policies__with_ecr()
self.api.create_role_and_policies(policies)
sleep(5) # to give time for AWS to sync up internally
self.api.create_project_with_container__gs_docker_codebuild()
self.api.code_build.build_start()
def setUp(self):
super().setUp()
self.github_org = 'filetrust'
self.source_version = 'master'
self.project_name = 'osbot-docker-codebuild'
self.build_spec = 'builds/osbot-jupyter/buildspec.yml'
self.docker_type = 'LINUX_CONTAINER'
self.docker_image = 'aws/codebuild/docker:18.09.0'
self.compute_type = 'BUILD_GENERAL1_LARGE'
self.delete_project = True
self.api = Create_Code_Build(project_name=self.project_name,
github_org=self.github_org,
source_version=self.source_version,
docker_type=self.docker_type,
docker_image=self.docker_image,
compute_type=self.compute_type,
build_spec=self.build_spec)
self.account_id = self.api.account_id
self.region = self.api.region
def setUp(self):
super().setUp()
#Deploy().setup()
self.project_name = 'OSBot-Jupyter'
self.iam = IAM()
self.account_id = self.iam.account_id()
self.region = self.iam.region()
self.github_org = 'filetrust'
self.source_version = 'master'
self.build_spec = 'buildspec.yml'
self.docker_type = 'LINUX_CONTAINER'
#self.docker_image = '{0}.dkr.ecr.eu-west-1.amazonaws.com/osbot-jupyter:latest'.format(self.account_id),
self.compute_type = 'BUILD_GENERAL1_MEDIUM'
#self.api = Create_Code_Build(project_name=self.project_name)
self.api = Create_Code_Build(project_name=self.project_name,
github_org=self.github_org,
source_version=self.source_version,
docker_type=self.docker_type,
compute_type=self.compute_type,
build_spec=self.build_spec)
class test_OSBot_Jupyter_Create_Code_Build(Test_Helper):
def setUp(self):
super().setUp()
#Deploy().setup()
self.project_name = 'OSBot-Jupyter'
self.iam = IAM()
self.account_id = self.iam.account_id()
self.region = self.iam.region()
self.github_org = 'filetrust'
self.source_version = 'master'
self.build_spec = 'buildspec.yml'
self.docker_type = 'LINUX_CONTAINER'
#self.docker_image = '{0}.dkr.ecr.eu-west-1.amazonaws.com/osbot-jupyter:latest'.format(self.account_id),
self.compute_type = 'BUILD_GENERAL1_MEDIUM'
#self.api = Create_Code_Build(project_name=self.project_name)
self.api = Create_Code_Build(project_name=self.project_name,
github_org=self.github_org,
source_version=self.source_version,
docker_type=self.docker_type,
compute_type=self.compute_type,
build_spec=self.build_spec)
#def create_project_with_container__osbot_jupyter(self):
# kvargs = {
# 'name' : self.api.project_name,
# 'source' : {'type' : 'GITHUB',
# 'location' : self.api.project_repo },
# 'artifacts' : {'type' : 'NO_ARTIFACTS' },
# 'environment' : {'type' : 'LINUX_CONTAINER' ,
# 'image' : '{0}.dkr.ecr.eu-west-1.amazonaws.com/osbot-jupyter:latest'.format(self.account_id) ,
# 'computeType' : 'BUILD_GENERAL1_SMALL' ,
# 'imagePullCredentialsType' : 'SERVICE_ROLE' },
# 'serviceRole' : self.api.service_role
# }
# return self.api.code_build.codebuild.create_project(**kvargs)
# this only needs to run once
def test_create_policies(self):
policies = self.api.policies__with_ecr_and_3_secrets()
self.api.create_role_and_policies(policies)
def test_create_code_build_and_trigger_first_build(self):
self.api.code_build.project_delete()
self.api.create_project_with_container__gs_docker_codebuild()
#self.create_project_with_container__osbot_jupyter()
#self.api.code_build.build_start()
def test_get_task_details(self):
from osbot_aws.apis.Logs import Logs
def find_starts(array, text):
return [item for item in array if item.startswith(text)]
def find_in(array, text):
return [item for item in array if text in item]
#build_id = 'OSBot-Jupyter:a553dda5-953a-41b8-ae91-e068cba4f56b'
result = self.api.code_build.project_builds_ids(self.api.project_name)
build_id = result.__next__() # get last one
build_info = self.api.code_build.build_info(build_id)
group_name = build_info.get('logs').get('groupName')
stream_name = build_info.get('logs').get('streamName')
#Dev.pprint(group_name,stream_name)
logs = Logs(group_name=group_name, stream_name=stream_name)
messages = logs.messages()
#ngrok_messages = find_starts(messages,'t=')
ngrok_url = find_in(
messages, 'name=command_line addr')[0].split('url=')[1].strip()
jupyter_token = find_in(messages,
'token=')[0].split('token=')[1].strip()
Dev.pprint("{0}?token={1}".format(ngrok_url, jupyter_token))
def __init__(self):
self.project_name = 'OSBot-gsheet-sync'
self.account_id = IAM().account_id()
self.api = Create_Code_Build(account_id=self.account_id,
project_name=self.project_name)
def setUp(self):
self.project_name = 'pbx-gs-python-utils'
self.account_id = IAM().account_id()
self.api = Create_Code_Build(account_id=self.account_id,
project_name=self.project_name)
def setUp(self):
self.project_name = 'osbot_aws'
self.account_id = IAM().account_id()
self.api = Create_Code_Build(account_id=self.account_id,
project_name=self.project_name)
class test_Create_Code_Build(TestCase):
def setUp(self):
self.project_name = 'osbot_aws'
self.account_id = IAM().account_id()
self.api = Create_Code_Build(account_id=self.account_id,
project_name=self.project_name)
# this needs to be updated to take into account the policies manually added to the arn:aws:iam::244560807427:role/osbot-aws
# via the AWS Console API
def test_create_polcies(self):
policies = {
"Cloud-Watch-Policy": {
"Version":
"2012-10-17",
"Statement": [{
"Sid":
"GsBotPolicy",
"Effect":
"Allow",
"Action": [
"logs:CreateLogGroup", "logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": [cloud_watch_arn]
}]
},
"Download_Image": {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:GetRepositoryPolicy", "ecr:DescribeRepositories",
"ecr:ListImages", "ecr:DescribeImages",
"ecr:BatchGetImage"
],
"Resource":
"*"
}]
}
}
# "Secret-Manager" : {
# "Version" : "2012-10-17",
# "Statement": [{ "Effect" : "Allow",
# "Action" : [ "secretsmanager:GetSecretValue","secretsmanager:DescribeSecret" ],
# "Resource": [ "arn:aws:secretsmanager:eu-west-2:244560807427:secret:slack-gs-bot-*" ,
# "arn:aws:secretsmanager:eu-west-2:244560807427:secret:elastic*" ,
# "arn:aws:secretsmanager:eu-west-2:244560807427:secret:gsuite*" ]}]},
# "Update_Lambda" : { "Version" : "2012-10-17",
# "Statement": [ { "Effect": "Allow", "Action" : ["s3:PutObject" ], "Resource": ["arn:aws:s3:::gs-lambda-tests/dinis/lambdas/gsbot_gsuite_lambdas_gdocs.zip"] },
# { "Effect": "Allow", "Action" : ["lambda:CreateFunction"], "Resource": ["arn:aws:lambda:eu-west-2:244560807427:function:gsbot_gsuite_lambdas_gdocs"] } ]}}
def test_create(self):
policies = self.api.policies__with_ecr()
self.api.create_role_and_policies(policies)
sleep(5) # to give time for AWS to sync up internally
self.api.create_project_with_container__gs_docker_codebuild()
self.api.code_build.build_start()
class test_Create_Code_Build(Test_Helper):
def setUp(self):
super().setUp()
self.github_org = 'filetrust'
self.source_version = 'master'
self.project_name = 'osbot-docker-codebuild'
self.build_spec = 'builds/osbot-jupyter/buildspec.yml'
self.docker_type = 'LINUX_CONTAINER'
self.docker_image = 'aws/codebuild/docker:18.09.0'
self.compute_type = 'BUILD_GENERAL1_LARGE'
self.delete_project = True
self.api = Create_Code_Build(project_name=self.project_name,
github_org=self.github_org,
source_version=self.source_version,
docker_type=self.docker_type,
docker_image=self.docker_image,
compute_type=self.compute_type,
build_spec=self.build_spec)
self.account_id = self.api.account_id
self.region = self.api.region
def test_check_setup(self):
assert self.api.account_id == "311800962295"
assert self.api.region == 'eu-west-1'
def test_create_all(self):
if self.delete_project:
self.api.delete_project_role_and_policies()
assert self.api.code_build.project_exists() is False
assert self.api.code_build.iam.role_exists() is False
expected_account_id = '311800962295'
expected_project_arn = f'arn:aws:codebuild:{self.region}:{self.account_id}:project/{self.project_name}'
expected_build_id_start = f'arn:aws:codebuild:{self.region}:{self.account_id}:build/{self.project_name}'
Assert(self.account_id).is_equal(
expected_account_id
) # confirm account was correctly set from profile_name
Assert(self.api.project_name).is_equal(
self.project_name) # confirm vars setup
Assert(self.api.project_repo).is_equal(
f'https://github.com/{self.github_org}/{self.project_name}'
) # confirm repo
policies = self.api.policies__for_docker_build()
self.api.create_role_and_policies(policies)
sleep(1) # to give time for AWS to sync up internally
# todo: figure out why the next line fails due to the AWS permission problem
# it is caused by some of IAM permissions not having been propagated
assert self.api.create_project_with_container__docker(
)['project']['arn'] == expected_project_arn
assert self.project_name in self.api.code_build.projects(
) # confirm project has been created
assert self.api.code_build.iam.role_exists(
) is True # confirm role has been created
assert self.api.code_build.build_start().startswith(
expected_build_id_start) # start build
# run just this part independently (if it fails above)
def test_just_create_project(self):
expected_project_arn = f'arn:aws:codebuild:{self.region}:{self.account_id}:project/{self.project_name}'
expected_build_id_start = f'arn:aws:codebuild:{self.region}:{self.account_id}:build/{self.project_name}'
assert self.api.create_project_with_container__docker(
)['project']['arn'] == expected_project_arn
assert self.project_name in self.api.code_build.projects(
) # confirm project has been created
assert self.api.code_build.iam.role_exists(
) is True # confirm role has been created
assert self.api.code_build.build_start().startswith(
expected_build_id_start) # start build
# use this to start a code build
def test_build_start(self):
build_id = self.api.code_build.build_start()
result = self.api.code_build.build_wait_for_completion(
build_id, max_attempts=100, log_status=True)
Dev.pprint(result)
def setUp(self):
self.project_name = 'OSBot-jira'
self.api = Create_Code_Build(project_name=self.project_name)
def setUp(self):
self.project_name = 'entityresolver'
self.profile_name = 'gs-detect-aws'
self.account_id = IAM().account_id(self.profile_name)
self.api = Create_Code_Build(account_id=self.account_id, project_name=self.project_name)