def client_boto3(self, service_name, profile_name=None, region_name=None): # todo: refactor with resource_boto3 try: profile_name = profile_name or AWS_Config( ).aws_session_profile_name() region_name = region_name or AWS_Config().aws_session_region_name() # profiles = get_session()._build_profile_map() # if profile_name in profiles: # session = boto3.Session(profile_name=profile_name, region_name=region_name) if profile_name in self.profiles( ): # seeing if this is a more efficient way to get the data session = boto3.Session( profile_name=profile_name, region_name=region_name ) # tried to pass this params but had side effects: , botocore_session=self.boto_session() return { 'status': 'ok', 'client': session.client(service_name=service_name), "session": session } return { 'status': 'ok', 'client': boto3.client(service_name=service_name, region_name=region_name) } except Exception as error: return {'status': 'error', 'data': '{0}'.format(error)}
def test_client_boto3(self): assert "Unknown service: 'aaaa'. Valid service names are:" in self.session.client_boto3( 'aaaa').get('data') AWS_Config().set_aws_session_profile_name('bad_profile') assert type( self.session.client_boto3('s3').get('client')).__name__ == 'S3' AWS_Config().set_aws_session_profile_name('default')
def _test_run_lambdas_in_multiple_accounts(self): from osbot_aws.apis.test_helpers.Temp_Lambda import Temp_Lambda AWS_Config().set_aws_session_profile_name('gs-detect-aws') AWS_Config().set_lambda_s3_bucket('gs-detect-lambda') with Temp_Lambda() as _: _.invoke_raw().get('status') == 'ok' AWS_Config().set_aws_session_profile_name('default')
def session( self, profile_name=None, region_name=None) -> Session: # todo: refactor with resource_boto3 profile_name = profile_name or AWS_Config().aws_session_profile_name() region_name = region_name or AWS_Config().aws_session_region_name() profiles = get_session()._build_profile_map() if profile_name in profiles: return boto3.Session(profile_name=profile_name, region_name=region_name) else: return boto3.Session()
def setUpClass(cls) -> None: #STS().check_current_session_credentials() cls.s3 = S3() cls.aws_config = AWS_Config() cls.account_id = cls.aws_config.aws_session_account_id() cls.s3_bucket = cls.aws_config.lambda_s3_bucket() cls.region = cls.aws_config.aws_session_region_name()
def __init__(self, layer_name='', runtimes=None, license_info=None, s3_bucket=None, s3_folder=None, description=None, version_number=None): self.layer_name = layer_name.replace('.', '-') #self.folders_mapping = folders_mapping or {} self.runtimes = runtimes or ['python3.8', 'python3.7', 'python3.6'] self.license_info = license_info or 'https://github.com/owasp-sbot/OSBot-AWS/blob/master/LICENSE' self.description = description or '' self.s3_bucket = s3_bucket or AWS_Config().lambda_s3_bucket() self.s3_folder = s3_folder or AWS_Config().lambda_s3_folder_layers() self.s3_key = f'{self.s3_folder}/{self.layer_name}.zip' self.version_number = version_number
def test_account_id(self): account_id_1 = self.iam.account_id('gs-detect-aws') # todo: rewrite since account_id doesn't take this parameter any more assert AWS_Config().aws_session_profile_name() == 'gs-detect-aws' self.iam._account_id = None self.iam._sts = None account_id_2 = self.iam.account_id('default') assert AWS_Config().aws_session_profile_name() == 'default' assert account_id_1 != account_id_2 self.iam._account_id = None self.iam._sts = None account_id_3 = self.iam.account_id() assert AWS_Config().aws_session_profile_name() == 'default' assert account_id_2 == account_id_3
def upload_dependency(target): s3 = S3() s3_bucket = AWS_Config().lambda_s3_bucket() s3_file = 'lambdas-dependencies/{0}.zip'.format(target) path_libs = Files.path_combine('../../../_lambda_dependencies/', target) if Files.not_exists(path_libs): raise Exception(f"In Lambda upload_dependency, could not find dependency for: {target} , which resolved to {path_libs}") s3.folder_upload(path_libs, s3_bucket, s3_file) return s3.file_exists(s3_bucket, s3_file)
def reset(self): if self.aws_lambda.s3_bucket is None: # if these values are not set self.aws_lambda.set_s3_bucket( AWS_Config().lambda_s3_bucket()) # use default values self.aws_lambda.set_s3_key( f'{AWS_Config().lambda_s3_folder_lambdas()}/{self.aws_lambda.original_name}.zip' ) # which are needed return self.aws_lambda.update_lambda_code( ) # to trigger the update (which will reset the lambda and force a cold start on next lambda invocation)
def __init__(self, file_name=None,s3_bucket=None, s3_prefix=None): self.file_name = file_name or f"temp_zip_file_{random_string_and_numbers()}" self.s3_bucket = AWS_Config().lambda_s3_bucket() self.s3_prefix = f'{AWS_Config().lambda_s3_folder_lambdas()}/unit_tests/temp_zips' self.s3_key = f'{self.s3_prefix}/{self.file_name}.zip' self.folder = None self.lambda_code = "def run(event, context): return 'hello {0}'.format(event.get('name'))" self.tmp_file = None self.create_temp_file()
def __init__(self, lambda_name=None, delete_on_exit=True): self.lambda_name = lambda_name or "temp_lambda_{0}".format(random_string_and_numbers()) self.aws_lambda = Lambda(self.lambda_name) self.tmp_folder = Temp_Folder_With_Lambda_File(self.lambda_name).create_temp_file() self.role_arn = Temp_Aws_Roles().for_lambda_invocation__role_arn() # todo: refactor to have option to create the role programatically (needs feature to wait for role to be available) self.create_log = None self.delete_on_exit = delete_on_exit self.s3_bucket = AWS_Config().lambda_s3_bucket() self.s3_key = 'unit_tests/lambdas/{0}.zip'.format(self.lambda_name) self.s3 = self.aws_lambda.s3()
def __init__(self, image_name, path_images=None, image_tag='latest') -> object: self.api_docker = API_Docker() self.ecr = ECR() self.aws_config = AWS_Config() self.image_name = image_name self.image_tag = image_tag self.path_images = path_images or path_combine(__file__, '../../images')
def setUpClass(cls) -> None: cls.aws_config = AWS_Config() cls.account_id = cls.aws_config.aws_session_account_id() cls.ecr = ECR() cls.repository_name = 'an_test_repository' cls.repository_tags = {'an_tag': 'an_value'} assert cls.ecr.repository_exists(cls.repository_name) is False cls.result_create = cls.ecr.repository_create(name=cls.repository_name, tags=cls.repository_tags) assert cls.ecr.repository_exists(cls.repository_name) is True
def __init__(self,lambda_name): self.lambda_name = lambda_name self.aws_lambda = Lambda(self.lambda_name) self.s3_bucket = AWS_Config().lambda_s3_bucket() self.s3_key = f'{AWS_Config().lambda_s3_folder_lambdas()}/{self.lambda_name}.zip' self.role_arn = Temp_Aws_Roles().for_lambda_invocation__role_arn() self.tmp_folder = Files.temp_folder('tmp_lambda_') (self.aws_lambda.set_s3_bucket (self.s3_bucket ) .set_s3_key (self.s3_key ) .set_role (self.role_arn ) .set_folder_code (self.tmp_folder ))
def resource_boto3(self, service_name, profile_name=None, region_name=None): # todo: refactor with client_boto3 try: profile_name = profile_name or AWS_Config( ).aws_session_profile_name() region_name = region_name or AWS_Config().aws_session_region_name() profiles = get_session()._build_profile_map() if profile_name in profiles: session = boto3.Session(profile_name=profile_name, region_name=region_name) return { 'status': 'ok', 'resource': session.resource(service_name=service_name) } return { 'status': 'ok', 'resource': boto3.resource(service_name=service_name) } except Exception as error: return {'status': 'error', 'data': '{0}'.format(error)}
def setUp(self): self.aws_config = AWS_Config() self.lambda_ = Lambda() self.s3 = S3() self.sts = STS() self.expected_account_id = '785217600689' self.expected_region = 'eu-west-1' self.expected_s3_prefix = 'lambdas' self.expected_role_name = None self.expected_s3_bucket = f'{self.expected_account_id}-osbot-{self.expected_s3_prefix}' self.expected_module = 'osbot_aws.lambdas.dev.hello_world' self.function_name = 'osbot_aws_lambdas_dev_hello_world' self.lambda_handler = run
def check_current_session_credentials( self, raise_exception=True ): # todo: see if there is a faster way to do this, at the moment it takes about 500ms which is quite a lot if AWS_Config().dev_skip_aws_key_check() == "True": return status_warning(message="check was skipped") result = self.check_aws_session() if result.get('status') == 'error': self.print_bad_credentials_exception(result.get('message')) #if exit_on_error: # todo: see better way to do this # self.end_process_bad_credentials_exception() if raise_exception: self.raise_bad_credentials_exception(result.get('message')) return result
def test_session(self): assert self.session.session().profile_name == AWS_Config( ).aws_session_profile_name() assert self.session.session().region_name == AWS_Config( ).aws_session_region_name() with Temp_User() as temp_user: iam = temp_user.iam user_info = iam.user_info() access_key = iam.user_access_key_create(wait_for_key_working=True) aws_access_key_id = access_key.get('AccessKeyId') aws_secret_access_key = access_key.get('SecretAccessKey') session = boto3.Session( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) user_identity = session.client('sts').get_caller_identity() assert user_identity.get('UserId') == user_info.get('UserId') assert user_identity.get('Arn') == user_info.get('Arn') self.result = user_identity
def __init__(self, cluster_name, image_name, subnet_id=None, security_group_id=None): self.ecs = ECS() self.ec2 = EC2() self.cloud_watch_logs = Cloud_Watch_Logs() self.aws_config = AWS_Config() # load config from env variables self.account_id = self.cloud_watch_logs.account_id self.region_name = self.cloud_watch_logs.region_name self.cluster_name = cluster_name self.image_name = image_name self.subnet_id = subnet_id self.security_group_id = security_group_id self.task_family = f"family__{self.image_name}" self.task_name = f'task__{self.cluster_name}' #self.iam_execution_role = f'fargate-execution-role_{self.region_name}_{self.task_family}' #self.iam_task_role = f'fargate-task-role_{self.region_name}_{self.task_family}' self.task_arn = None
def __init__(self, bot_name= None, profile_name = None, account_id=None, region_name=None, lambda_s3_bucket=None, lambda_role_name=None): aws_config = AWS_Config() if bot_name : aws_config.set_bot_name (bot_name) if profile_name : aws_config.set_aws_session_profile_name(profile_name ) if account_id : aws_config.set_aws_session_account_id (account_id ) if region_name : aws_config.set_aws_session_region_name (region_name ) if lambda_s3_bucket : aws_config.set_lambda_s3_bucket (lambda_s3_bucket) if lambda_role_name : aws_config.set_lambda_role_name (lambda_role_name) self.bot_name = aws_config.bot_name() self.profile_name = aws_config.aws_session_profile_name() self.region_name = aws_config.aws_session_region_name() self.account_id = aws_config.aws_session_account_id() self.s3_bucket_lambdas = aws_config.lambda_s3_bucket() self.lambda_role_name = aws_config.lambda_role_name() self.lambda_role_arn = f"arn:aws:iam::{self.account_id}:role/{self.lambda_role_name}" self.s3 = S3()
def load_dependency(target): if os.getenv('AWS_REGION') is None: return from osbot_aws.apis.S3 import S3 import shutil import sys s3 = S3() s3_bucket = AWS_Config().lambda_s3_bucket() s3_key = 'lambdas-dependencies/{0}.zip'.format(target) tmp_dir = Files.path_combine('/tmp/lambdas-dependencies', target) #return s3.file_exists(s3_bucket,s3_key) if s3.file_exists(s3_bucket,s3_key) is False: raise Exception("In Lambda load_dependency, could not find dependency for: {0}".format(target)) if file_not_exists(tmp_dir): # download dependency zip_file = s3.file_download(s3_bucket, s3_key,False) # download zip file with dependencies shutil.unpack_archive(zip_file, extract_dir = tmp_dir) # unpack them if tmp_dir not in sys.path: # if not currently in the path sys.path.append(tmp_dir) # add tmp_dir to the path that python uses to check for dependencies return Files.exists(tmp_dir)
def setUp(self) -> None: self.deploy = Deploy(run) #Test_Helper().check_aws_token() self.aws_config = AWS_Config()
def setUp(self): self.account_id = AWS_Config().aws_session_account_id() self.iam_policy = IAM_Policy()
def test__init__(self): assert self.osbot_setup.profile_name == AWS_Config().aws_session_profile_name()
import pytest from osbot_aws.apis.STS import STS from osbot_utils.testing.Catch import Catch from osbot_utils.utils.Json import json_to_str from osbot_utils.utils.Misc import wait from osbot_aws.AWS_Config import AWS_Config from osbot_aws.helpers.Test_Helper import Test_Helper from osbot_aws.apis.IAM import IAM from osbot_utils.utils.Assert import Assert from osbot_utils.utils.Dev import pprint account_id = AWS_Config().aws_session_account_id() delete_created = True test_user = '******' test_user_arn = 'arn:aws:iam::{0}:user/test_user'.format(account_id) test_role = 'test_role' test_role_arn = 'arn:aws:iam::{0}:role/test_role'.format(account_id) policy_document = {'Statement': [ { 'Action' : 'sts:AssumeRole', 'Effect' : 'Allow', 'Principal': { 'Service': 'codebuild.amazonaws.com'}}]} class Test_IAM(Test_Helper): @classmethod def setUpClass(cls):
def test_for_lambda_invocation__role_arn(self): account_id = AWS_Config().aws_session_account_id() role_arn = self.temp_aws_roles.for_lambda_invocation__role_arn() assert role_arn == f"arn:aws:iam::{account_id}:role/temp_role_for_lambda_invocation"
def setUp(self) -> None: self.deploy = Deploy_Lambda(self.lambda_function) self.aws_config = AWS_Config() # Test_Helper().check_aws_token() pass
def test_image_repository(self): aws_config = AWS_Config() account_id = aws_config.aws_session_account_id() region = aws_config.aws_session_region_name() assert self._.image_repository( ) == f'{account_id}.dkr.ecr.{region}.amazonaws.com/{self.image_name}'
def for_lambda_invocation__role_arn(self): account_id = AWS_Config().aws_session_account_id() role_name = self.role_name__for_lambda_invocation return f"arn:aws:iam::{account_id}:role/{role_name}"
def setUp(self): self.temp_role_name = 'test_IAM_Role__temp_role' self.iam_role = IAM_Role(role_name=self.temp_role_name) with AWS_Config() as aws_config: self.account_id = aws_config.aws_session_account_id() self.region = aws_config.aws_session_region_name()