def create_data_bucket() -> str:
region = get_aws_region()
account_id = get_aws_account_id()
bucket_name = random_suffix_name(f"ack-data-bucket-{region}-{account_id}",
63)
s3 = boto3.client("s3", region_name=region)
if region == "us-east-1":
s3.create_bucket(Bucket=bucket_name)
else:
s3.create_bucket(
Bucket=bucket_name,
CreateBucketConfiguration={'LocationConstraint': region})
logging.info(f"Created SageMaker data bucket {bucket_name}")
s3_resource = boto3.resource("s3", region_name=region)
source_bucket = s3_resource.Bucket(SAGEMAKER_SOURCE_DATA_BUCKET)
destination_bucket = s3_resource.Bucket(bucket_name)
duplicate_s3_contents(source_bucket, destination_bucket)
logging.info(f"Synced data bucket")
return bucket_name
def create_execution_role() -> str:
region = get_aws_region()
role_name = random_suffix_name(f"ack-sagemaker-execution-role", 63)
iam = boto3.client("iam", region_name=region)
iam.create_role(
RoleName=role_name,
AssumeRolePolicyDocument=json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "sagemaker.amazonaws.com"
},
"Action": "sts:AssumeRole"
}]
}),
Description=
"SageMaker execution role for ACK integration and canary tests")
iam.attach_role_policy(
RoleName=role_name,
PolicyArn='arn:aws:iam::aws:policy/AmazonSageMakerFullAccess')
iam.attach_role_policy(
RoleName=role_name,
PolicyArn='arn:aws:iam::aws:policy/AmazonS3FullAccess')
iam_resource = iam.get_role(RoleName=role_name)
resource_arn = iam_resource['Role']['Arn']
logging.info(f"Created SageMaker execution role {resource_arn}")
return resource_arn
def delete_vpc(vpc_id: str):
region = get_aws_region()
ec2 = boto3.client("ec2", region_name=region)
ec2.delete_vpc(VpcId=vpc_id)
logging.info(f"Deleted VPC {vpc_id}")
def delete_subnet(subnet_id: str):
region = get_aws_region()
ec2 = boto3.client("ec2", region_name=region)
ec2.delete_subnet(SubnetId=subnet_id)
logging.info(f"Deleted VPC Subnet {subnet_id}")
def create_subnet(vpc_id: str) -> str:
region = get_aws_region()
ec2 = boto3.client("ec2", region_name=region)
resp = ec2.create_subnet(
CidrBlock=VPC_SUBNET_CIDR_BLOCK,
VpcId=vpc_id,
)
subnet_id = resp['Subnet']['SubnetId']
# TODO(jaypipes): Put a proper waiter here...
time.sleep(3)
subnets = ec2.describe_subnets(SubnetIds=[subnet_id])
if len(subnets['Subnets']) != 1:
raise RuntimeError(
f"failed to describe subnet we just created '{subnet_id}'", )
subnet = subnets['Subnets'][0]
subnet_state = subnet['State']
if subnet_state != "available":
raise RuntimeError(
f"Subnet we just created '{subnet_id}' is not available. current state: {subnet_state}",
)
logging.info(f"Created VPC Subnet {subnet_id}")
return subnet_id
def create_vpc() -> str:
region = get_aws_region()
ec2 = boto3.client("ec2", region_name=region)
logging.debug(f"Creating VPC with CIDR {VPC_CIDR_BLOCK}")
resp = ec2.create_vpc(CidrBlock=VPC_CIDR_BLOCK, )
vpc_id = resp['Vpc']['VpcId']
# TODO(jaypipes): Put a proper waiter here...
time.sleep(3)
vpcs = ec2.describe_vpcs(VpcIds=[vpc_id])
if len(vpcs['Vpcs']) != 1:
raise RuntimeError(
f"failed to describe VPC we just created '{vpc_id}'", )
vpc = vpcs['Vpcs'][0]
vpc_state = vpc['State']
if vpc_state != "available":
raise RuntimeError(
f"VPC we just created '{vpc_id}' is not available. current state: {vpc_state}",
)
logging.info(f"Created VPC {vpc_id}")
return vpc_id
def delete_data_bucket(bucket_name: str):
region = get_aws_region()
s3_resource = boto3.resource("s3", region_name=region)
bucket = s3_resource.Bucket(bucket_name)
bucket.objects.all().delete()
bucket.delete()
logging.info(f"Deleted data bucket {bucket_name}")
def delete_execution_role(role_arn: str):
region = get_aws_region()
iam = boto3.client("iam", region_name=region)
role_name = re.match(IAM_ROLE_ARN_REGEX, role_arn).group(1)
managedPolicy = iam.list_attached_role_policies(RoleName=role_name)
for each in managedPolicy['AttachedPolicies']:
iam.detach_role_policy(RoleName=role_name, PolicyArn=each['PolicyArn'])
inlinePolicy = iam.list_role_policies(RoleName=role_name)
for each in inlinePolicy['PolicyNames']:
iam.delete_role_policy(RoleName=role_name, PolicyName=each)
instanceProfiles = iam.list_instance_profiles_for_role(RoleName=role_name)
for each in instanceProfiles['InstanceProfiles']:
iam.remove_role_from_instance_profile(
RoleName=role_name,
InstanceProfileName=each['InstanceProfileName'])
iam.delete_role(RoleName=role_name)
logging.info(f"Deleted SageMaker execution role {role_name}")
def create_execution_role() -> str:
region = get_aws_region()
role_name = random_suffix_name(f"ack-sagemaker-execution-role", 63)
iam = boto3.client("iam", region_name=region)
iam.create_role(
RoleName=role_name,
AssumeRolePolicyDocument=json.dumps({
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": "sagemaker.amazonaws.com"
},
"Action": "sts:AssumeRole"
}]
}),
Description=
"SageMaker execution role for ACK integration and canary tests")
iam.attach_role_policy(
RoleName=role_name,
PolicyArn='arn:aws:iam::aws:policy/AmazonSageMakerFullAccess')
iam.attach_role_policy(
RoleName=role_name,
PolicyArn='arn:aws:iam::aws:policy/AmazonS3FullAccess')
iam_resource = iam.get_role(RoleName=role_name)
resource_arn = iam_resource['Role']['Arn']
# There appears to be a delay in role availability after role creation
# resulting in failure that role is not present. So adding a delay
# to allow for the role to become available
time.sleep(10)
logging.info(f"Created SageMaker execution role {resource_arn}")
return resource_arn
def create_security_group() -> str:
region = get_aws_region()
account_id = get_aws_account_id()
ec2 = boto3.client("ec2")
vpc_response = ec2.describe_vpcs(Filters=[{
"Name": "isDefault",
"Values": ["true"]
}])
if len(vpc_response['Vpcs']) == 0:
raise ValueError(
f"Default VPC not found for account {account_id} in region {region}"
)
default_vpc_id = vpc_response['Vpcs'][0]['VpcId']
sg_name = random_suffix_name("ack-security-group", 32)
sg_description = "Security group for ACK ElastiCache tests"
sg_response = ec2.create_security_group(GroupName=sg_name,
VpcId=default_vpc_id,
Description=sg_description)
logging.info(f"Created VPC Security Group {sg_response['GroupId']}")
return sg_response['GroupId']