def create_data_bucket() -> str: region = get_region() account_id = get_account_id() bucket_name = resources.random_suffix_name( f"ack-data-bucket-{region}-{account_id}", 63) s3 = boto3.client("s3", region_name=region) if region == "us-east-1": s3.create_bucket(Bucket=bucket_name) else: s3.create_bucket( Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": region}) logging.info(f"Created SageMaker data bucket {bucket_name}") s3_resource = boto3.resource("s3", region_name=region) source_bucket = s3_resource.Bucket(SAGEMAKER_SOURCE_DATA_BUCKET) destination_bucket = s3_resource.Bucket(bucket_name) duplicate_bucket_contents(source_bucket, destination_bucket) logging.info(f"Synced data bucket") return bucket_name
def create_data_bucket() -> str: region = get_region() account_id = get_account_id() bucket_name = resources.random_suffix_name( f"ack-data-bucket-{region}-{account_id}", 63) s3 = boto3.client("s3", region_name=region) if region == "us-east-1": s3.create_bucket(Bucket=bucket_name) else: s3.create_bucket( Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": region}) logging.info(f"Created SageMaker data bucket {bucket_name}") s3_resource = boto3.resource("s3", region_name=region) source_bucket = s3_resource.Bucket(SAGEMAKER_SOURCE_DATA_BUCKET) destination_bucket = s3_resource.Bucket(bucket_name) temp_dir = "/tmp/ack_s3_data" # awscli is not installed in test-infra container hence use boto3 to copy in us-west-2 if region == "us-west-2": duplicate_bucket_contents(source_bucket, destination_bucket) # above method does an async copy # TODO: find a way to remove random wait time.sleep(180) else: # workaround to copy if buckets are across regions # TODO: check if there is a better way and merge to test-infra subprocess.call(["mkdir", f"{temp_dir}"]) subprocess.call([ "aws", "s3", "sync", f"s3://{SAGEMAKER_SOURCE_DATA_BUCKET}", f"./{temp_dir}/", "--quiet", ]) subprocess.call([ "aws", "s3", "sync", f"./{temp_dir}/", f"s3://{bucket_name}", "--quiet" ]) logging.info(f"Synced data bucket") return bucket_name
def authorizer_resource(api_resource): random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) authorizer_resource_name = test_resource_values[ 'AUTHORIZER_NAME'] + f'-{random_suffix}' test_resource_values['AUTHORIZER_NAME'] = authorizer_resource_name authorizer_uri = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations' test_resource_values["AUTHORIZER_URI"] = authorizer_uri authorizer_ref, authorizer_data = helper.authorizer_ref_and_data( authorizer_resource_name=authorizer_resource_name, replacement_values=test_resource_values) if k8s.get_resource_exists(authorizer_ref): raise Exception( f"expected {authorizer_resource_name} to not exist. Did previous test cleanup?" ) logging.debug( f"apigatewayv2 authorizer resource. name: {authorizer_resource_name}, data: {authorizer_data}" ) k8s.create_custom_resource(authorizer_ref, authorizer_data) cr = k8s.wait_resource_consumed_by_controller(authorizer_ref) assert cr is not None assert k8s.get_resource_exists(authorizer_ref) authorizer_id = cr['status']['authorizerID'] test_resource_values['AUTHORIZER_ID'] = authorizer_id # add permissions for apigateway to invoke authorizer lambda authorizer_arn = "arn:aws:execute-api:{region}:{account}:{api_id}/authorizers/{authorizer_id}".format( region=get_region(), account=get_account_id(), api_id=test_resource_values['API_ID'], authorizer_id=authorizer_id) lambda_client = boto3.client("lambda") lambda_client.add_permission( FunctionName=get_bootstrap_resources().AuthorizerFunctionName, StatementId= f'apigatewayv2-authorizer-invoke-permissions-{random_suffix}', Action='lambda:InvokeFunction', Principal='apigateway.amazonaws.com', SourceArn=authorizer_arn) yield authorizer_ref, cr k8s.delete_custom_resource(authorizer_ref)
def service_bootstrap() -> Resources: logging.getLogger().setLevel(logging.INFO) region = get_region() account_id = get_account_id() bucket_name = f"ack-data-bucket-{region}-{account_id}" resources = TestBootstrapResources( DataBucket=Bucket(bucket_name), ExecutionRole=Role( "ack-sagemaker-execution-role", "sagemaker.amazonaws.com", managed_policies=[ "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess", "arn:aws:iam::aws:policy/AmazonS3FullAccess", ], ), ) try: resources.bootstrap() sync_data_bucket(resources.DataBucket) except BootstrapFailureException as ex: exit(254) return resources
def create_security_group() -> str: region = get_region() account_id = get_account_id() ec2 = boto3.client("ec2") vpc_response = ec2.describe_vpcs(Filters=[{ "Name": "isDefault", "Values": ["true"] }]) if len(vpc_response['Vpcs']) == 0: raise ValueError( f"Default VPC not found for account {account_id} in region {region}" ) default_vpc_id = vpc_response['Vpcs'][0]['VpcId'] sg_name = random_suffix_name("ack-security-group", 32) sg_description = "Security group for ACK ElastiCache tests" sg_response = ec2.create_security_group(GroupName=sg_name, VpcId=default_vpc_id, Description=sg_description) logging.info(f"Created VPC Security Group {sg_response['GroupId']}") return sg_response['GroupId']
def authorizer_resource(api_resource): authorizer_resource_name = random_suffix_name(test_resource_values['AUTHORIZER_NAME'], 25) test_resource_values['AUTHORIZER_NAME'] = authorizer_resource_name authorizer_uri = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations' test_resource_values["AUTHORIZER_URI"] = authorizer_uri authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(authorizer_resource_name=authorizer_resource_name, replacement_values=test_resource_values) k8s.create_custom_resource(authorizer_ref, authorizer_data) time.sleep(CREATE_WAIT_AFTER_SECONDS) assert k8s.wait_on_condition(authorizer_ref, "ACK.ResourceSynced", "True", wait_periods=10) cr = k8s.get_resource(authorizer_ref) assert cr is not None authorizer_id = cr['status']['authorizerID'] test_resource_values['AUTHORIZER_ID'] = authorizer_id # add permissions for apigateway to invoke authorizer lambda authorizer_arn = "arn:aws:execute-api:{region}:{account}:{api_id}/authorizers/{authorizer_id}".format( region=get_region(), account=get_account_id(), api_id=test_resource_values['API_ID'], authorizer_id=authorizer_id ) lambda_client = boto3.client("lambda") function_name = get_bootstrap_resources().AuthorizerFunctionName statement_id = random_suffix_name('invoke-permission', 25) lambda_client.add_permission(FunctionName=function_name, StatementId=statement_id, Action='lambda:InvokeFunction', Principal='apigateway.amazonaws.com', SourceArn=authorizer_arn) yield authorizer_ref, cr lambda_client.remove_permission(FunctionName=function_name, StatementId=statement_id) k8s.delete_custom_resource(authorizer_ref)