def create_data_bucket() -> str:
region = get_region()
account_id = get_account_id()
bucket_name = resources.random_suffix_name(
f"ack-data-bucket-{region}-{account_id}", 63)
s3 = boto3.client("s3", region_name=region)
if region == "us-east-1":
s3.create_bucket(Bucket=bucket_name)
else:
s3.create_bucket(
Bucket=bucket_name,
CreateBucketConfiguration={"LocationConstraint": region})
logging.info(f"Created SageMaker data bucket {bucket_name}")
s3_resource = boto3.resource("s3", region_name=region)
source_bucket = s3_resource.Bucket(SAGEMAKER_SOURCE_DATA_BUCKET)
destination_bucket = s3_resource.Bucket(bucket_name)
duplicate_bucket_contents(source_bucket, destination_bucket)
logging.info(f"Synced data bucket")
return bucket_name
def create_data_bucket() -> str:
region = get_region()
account_id = get_account_id()
bucket_name = resources.random_suffix_name(
f"ack-data-bucket-{region}-{account_id}", 63)
s3 = boto3.client("s3", region_name=region)
if region == "us-east-1":
s3.create_bucket(Bucket=bucket_name)
else:
s3.create_bucket(
Bucket=bucket_name,
CreateBucketConfiguration={"LocationConstraint": region})
logging.info(f"Created SageMaker data bucket {bucket_name}")
s3_resource = boto3.resource("s3", region_name=region)
source_bucket = s3_resource.Bucket(SAGEMAKER_SOURCE_DATA_BUCKET)
destination_bucket = s3_resource.Bucket(bucket_name)
temp_dir = "/tmp/ack_s3_data"
# awscli is not installed in test-infra container hence use boto3 to copy in us-west-2
if region == "us-west-2":
duplicate_bucket_contents(source_bucket, destination_bucket)
# above method does an async copy
# TODO: find a way to remove random wait
time.sleep(180)
else:
# workaround to copy if buckets are across regions
# TODO: check if there is a better way and merge to test-infra
subprocess.call(["mkdir", f"{temp_dir}"])
subprocess.call([
"aws",
"s3",
"sync",
f"s3://{SAGEMAKER_SOURCE_DATA_BUCKET}",
f"./{temp_dir}/",
"--quiet",
])
subprocess.call([
"aws", "s3", "sync", f"./{temp_dir}/", f"s3://{bucket_name}",
"--quiet"
])
logging.info(f"Synced data bucket")
return bucket_name
def authorizer_resource(api_resource):
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
authorizer_resource_name = test_resource_values[
'AUTHORIZER_NAME'] + f'-{random_suffix}'
test_resource_values['AUTHORIZER_NAME'] = authorizer_resource_name
authorizer_uri = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations'
test_resource_values["AUTHORIZER_URI"] = authorizer_uri
authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(
authorizer_resource_name=authorizer_resource_name,
replacement_values=test_resource_values)
if k8s.get_resource_exists(authorizer_ref):
raise Exception(
f"expected {authorizer_resource_name} to not exist. Did previous test cleanup?"
)
logging.debug(
f"apigatewayv2 authorizer resource. name: {authorizer_resource_name}, data: {authorizer_data}"
)
k8s.create_custom_resource(authorizer_ref, authorizer_data)
cr = k8s.wait_resource_consumed_by_controller(authorizer_ref)
assert cr is not None
assert k8s.get_resource_exists(authorizer_ref)
authorizer_id = cr['status']['authorizerID']
test_resource_values['AUTHORIZER_ID'] = authorizer_id
# add permissions for apigateway to invoke authorizer lambda
authorizer_arn = "arn:aws:execute-api:{region}:{account}:{api_id}/authorizers/{authorizer_id}".format(
region=get_region(),
account=get_account_id(),
api_id=test_resource_values['API_ID'],
authorizer_id=authorizer_id)
lambda_client = boto3.client("lambda")
lambda_client.add_permission(
FunctionName=get_bootstrap_resources().AuthorizerFunctionName,
StatementId=
f'apigatewayv2-authorizer-invoke-permissions-{random_suffix}',
Action='lambda:InvokeFunction',
Principal='apigateway.amazonaws.com',
SourceArn=authorizer_arn)
yield authorizer_ref, cr
k8s.delete_custom_resource(authorizer_ref)
def service_bootstrap() -> Resources:
logging.getLogger().setLevel(logging.INFO)
region = get_region()
account_id = get_account_id()
bucket_name = f"ack-data-bucket-{region}-{account_id}"
resources = TestBootstrapResources(
DataBucket=Bucket(bucket_name),
ExecutionRole=Role(
"ack-sagemaker-execution-role",
"sagemaker.amazonaws.com",
managed_policies=[
"arn:aws:iam::aws:policy/AmazonSageMakerFullAccess",
"arn:aws:iam::aws:policy/AmazonS3FullAccess",
],
),
)
try:
resources.bootstrap()
sync_data_bucket(resources.DataBucket)
except BootstrapFailureException as ex:
exit(254)
return resources
def create_security_group() -> str:
region = get_region()
account_id = get_account_id()
ec2 = boto3.client("ec2")
vpc_response = ec2.describe_vpcs(Filters=[{
"Name": "isDefault",
"Values": ["true"]
}])
if len(vpc_response['Vpcs']) == 0:
raise ValueError(
f"Default VPC not found for account {account_id} in region {region}"
)
default_vpc_id = vpc_response['Vpcs'][0]['VpcId']
sg_name = random_suffix_name("ack-security-group", 32)
sg_description = "Security group for ACK ElastiCache tests"
sg_response = ec2.create_security_group(GroupName=sg_name,
VpcId=default_vpc_id,
Description=sg_description)
logging.info(f"Created VPC Security Group {sg_response['GroupId']}")
return sg_response['GroupId']
def authorizer_resource(api_resource):
authorizer_resource_name = random_suffix_name(test_resource_values['AUTHORIZER_NAME'], 25)
test_resource_values['AUTHORIZER_NAME'] = authorizer_resource_name
authorizer_uri = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations'
test_resource_values["AUTHORIZER_URI"] = authorizer_uri
authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(authorizer_resource_name=authorizer_resource_name,
replacement_values=test_resource_values)
k8s.create_custom_resource(authorizer_ref, authorizer_data)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
assert k8s.wait_on_condition(authorizer_ref, "ACK.ResourceSynced", "True", wait_periods=10)
cr = k8s.get_resource(authorizer_ref)
assert cr is not None
authorizer_id = cr['status']['authorizerID']
test_resource_values['AUTHORIZER_ID'] = authorizer_id
# add permissions for apigateway to invoke authorizer lambda
authorizer_arn = "arn:aws:execute-api:{region}:{account}:{api_id}/authorizers/{authorizer_id}".format(
region=get_region(),
account=get_account_id(),
api_id=test_resource_values['API_ID'],
authorizer_id=authorizer_id
)
lambda_client = boto3.client("lambda")
function_name = get_bootstrap_resources().AuthorizerFunctionName
statement_id = random_suffix_name('invoke-permission', 25)
lambda_client.add_permission(FunctionName=function_name,
StatementId=statement_id,
Action='lambda:InvokeFunction',
Principal='apigateway.amazonaws.com',
SourceArn=authorizer_arn)
yield authorizer_ref, cr
lambda_client.remove_permission(FunctionName=function_name, StatementId=statement_id)
k8s.delete_custom_resource(authorizer_ref)