def __init__(self, scope: cdk.Construct, construct_id: str,
stack_log_level: str, sales_lambda_ap_name: str,
sales_lambda_consumer_role, s3_object_lambda_fn_role,
sales_event_bkt, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Create Admin Access Point
# Admin can access the Bucket ARN - DO NOT NEED ACCESS POINTs
# Create lambda Access Point
lambda_s3_ap_prefix = "store_events"
lambda_s3_access_point_policy_doc = {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Principal": {
"AWS": [
f"{sales_lambda_consumer_role.role_arn}",
f"{s3_object_lambda_fn_role.role_arn}",
]
},
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource":
f"arn:aws:s3:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:accesspoint/{sales_lambda_ap_name}/object/{lambda_s3_ap_prefix}/*"
}]
}
self.lambda_consumer_ap = _s3.CfnAccessPoint(
self,
"lambdaConsumerAccessPoint",
bucket=sales_event_bkt.bucket_name,
name=f"{sales_lambda_ap_name}",
policy=lambda_s3_access_point_policy_doc)
###########################################
################# OUTPUTS #################
###########################################
output_0 = cdk.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = cdk.CfnOutput(
self,
"LambdaConsumerAccessPointArn",
value=
f"arn:aws:s3:{cdk.Aws.REGION}:{cdk.Aws.ACCOUNT_ID}:accesspoint/{sales_lambda_ap_name}",
description=f"The Lambda consumer access point bucket ARN")
def __init__(
self,
scope: core.Construct,
id: str,
infra: RtspBaseResourcesConstruct,
) -> None:
super().__init__(scope, id)
# Create S3 Object Lambda
self.normalize_function = RtspNormalizeImageAccessPointFunction(
self, 'NormalizeImage', infra=infra)
self.s3_accesspoint = s3.CfnAccessPoint(
self,
'AccessPoint',
bucket=infra.bucket.bucket_name,
name='rtsp-normalized-image',
vpc_configuration=s3.CfnAccessPoint.VpcConfigurationProperty(
vpc_id=infra.landing_zone.vpc.vpc_id))
self.lambda_accesspoint = s3ol.CfnAccessPoint(
self,
'Lambda-AccessPoint',
name='rtsp-normalized-image-ap'.lower(),
object_lambda_configuration=s3ol.CfnAccessPoint.
ObjectLambdaConfigurationProperty(
supporting_access_point=self.s3_accesspoint.get_att(
'Arn').to_string(),
cloud_watch_metrics_enabled=True,
transformation_configurations=[
s3ol.CfnAccessPoint.TransformationConfigurationProperty(
actions=['GetObject'],
content_transformation={
'AwsLambda': {
'FunctionArn':
self.normalize_function.function.function_arn
},
})
]))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# CloudFormation Parameters
glue_db_name = core.CfnParameter(
self,
"GlueDatabaseName",
type="String",
description="Glue Database where the Table belongs.",
allowed_pattern="[\w-]+",
)
glue_table_name = core.CfnParameter(
self,
"GlueTableName",
type="String",
description="Glue Table where access will be granted.",
allowed_pattern="[\w-]+",
)
grantee_role_arn = core.CfnParameter(
self,
"GranteeIAMRoleARN",
type="String",
description="IAM Role's ARN.",
allowed_pattern=
"arn:(aws[a-zA-Z-]*)?:iam::\d{12}:role\/?[a-zA-Z0-9_+=,.@\-]+")
grantee_vpc = core.CfnParameter(
self,
"GranteeVPC",
type="String",
description=
"VPC ID from where the S3 access point will be accessed.",
allowed_pattern="vpc-[a-zA-Z0-9]+")
is_lakeformation = core.CfnParameter(
self,
"LakeFormationParam",
type="String",
description=
"If Lake Formation is used, the stack must be deployed using an IAM role with Lake Formation Admin permissions.",
allowed_values=["Yes", "No"])
# CloudFormation Parameter Groups
self.template_options.description = "\
This template deploys an S3 Access Point which provides a given IAM Role \
access to the underlying data location for a given Glue Table.\n\
Main use case for this template is to grant an ETL process in another AWS Account, \
access to the S3 objects (e.g., Parquet files) associated to a Glue Table."
self.template_options.metadata = {
"AWS::CloudFormation::Interface": {
"License":
"MIT-0",
"ParameterGroups": [{
"Label": {
"default": "Lake Formation (Producer Account)"
},
"Parameters": [is_lakeformation.logical_id]
}, {
"Label": {
"default":
"Source Data Catalog Resource (Producer Account)"
},
"Parameters":
[glue_db_name.logical_id, glue_table_name.logical_id]
}, {
"Label": {
"default": "Grantee IAM Role (Consumer Account)"
},
"Parameters":
[grantee_role_arn.logical_id, grantee_vpc.logical_id]
}],
"ParameterLabels": {
is_lakeformation.logical_id: {
"default":
"Are data permissions managed by Lake Formation?"
},
glue_db_name.logical_id: {
"default": "What is the Glue DB Name for the Table?"
},
glue_table_name.logical_id: {
"default": "What is the Glue Table Name?"
},
grantee_role_arn.logical_id: {
"default": "What is the ARN of the IAM Role?"
},
grantee_vpc.logical_id: {
"default":
"What VPC will be used to access the S3 Access Point?"
}
}
}
}
is_lakeformation_condition = core.CfnCondition(
self,
"IsLakeFormation",
expression=core.Fn.condition_equals("Yes", is_lakeformation))
# Create S3 Access Point to share dataset objects
grantee_role = iam.Role.from_role_arn(self, "GranteeIAMRole",
grantee_role_arn.value_as_string)
glue_table_arn = f"arn:aws:glue:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:table/{glue_db_name.value_as_string}/{glue_table_name.value_as_string}"
glue_table = glue.Table.from_table_arn(self,
"GlueTable",
table_arn=glue_table_arn)
# Invoke Lambda to obtain S3 bucket and S3 prefix from Glue Table
get_s3_from_table_execution_role = iam.Role(
self,
"GetS3FromTableServiceRole",
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSGlueServiceRole")
])
lf_permission = lf.CfnPermissions(
self,
"LFPermissionForLambda",
data_lake_principal=lf.CfnPermissions.DataLakePrincipalProperty(
data_lake_principal_identifier=get_s3_from_table_execution_role
.role_arn),
resource=lf.CfnPermissions.ResourceProperty(
table_resource=lf.CfnPermissions.TableResourceProperty(
name=glue_table_name.value_as_string,
database_name=glue_db_name.value_as_string)),
permissions=["DESCRIBE"])
lf_permission.apply_removal_policy(core.RemovalPolicy.DESTROY,
apply_to_update_replace_policy=True)
lf_permission.node.add_dependency(get_s3_from_table_execution_role)
lf_permission.cfn_options.condition = is_lakeformation_condition
lf_wait_condition_handle = cfn.CfnWaitConditionHandle(
self, "LFWaitConditionHandle")
lf_wait_condition_handle.add_metadata(
"WaitForLFPermissionIfExists",
core.Fn.condition_if(is_lakeformation_condition.logical_id,
lf_permission.logical_id, ""))
with open("lambda/get_s3_from_table.py", encoding="utf8") as fp:
get_s3_from_table_code = fp.read()
get_s3_from_table_fn = _lambda.Function(
self,
"GetS3FromTableHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode.from_inline(get_s3_from_table_code),
handler="index.handler",
role=get_s3_from_table_execution_role,
timeout=core.Duration.seconds(600))
get_s3_from_table = core.CustomResource(
self,
"GetS3FromTable",
service_token=get_s3_from_table_fn.function_arn,
resource_type="Custom::GetS3FromTable",
properties={
"GlueDatabase": glue_db_name.value_as_string,
"GlueTable": glue_table_name.value_as_string
})
get_s3_from_table.node.add_dependency(lf_wait_condition_handle)
table_bucket = get_s3_from_table.get_att_string("TableBucket")
table_prefix = get_s3_from_table.get_att_string("TablePrefix")
# Create S3 Access Point
table_name_normalized = core.Fn.join(
"-", core.Fn.split("_", glue_table_name.value_as_string))
random_suffix = core.Fn.select(
0,
core.Fn.split(
"-", core.Fn.select(2, core.Fn.split("/", core.Aws.STACK_ID))))
s3_accesspoint_name = f"{table_name_normalized}-{random_suffix}"
s3_accesspoint_arn = f"arn:aws:s3:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:accesspoint/{s3_accesspoint_name}"
glue_table_accesspoint_path = f"{s3_accesspoint_arn}/object/{table_prefix}"
# s3_accesspoint_block_config = s3.CfnAccessPoint.PublicAccessBlockConfigurationProperty(block_public_acls=True, block_public_policy=True, ignore_public_acls=True, restrict_public_buckets=True)
s3_accesspoint_policy = iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
principals=[iam.ArnPrincipal(arn=grantee_role.role_arn)],
actions=["s3:GetObject*"],
resources=[f"{glue_table_accesspoint_path}*"]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
principals=[iam.ArnPrincipal(arn=grantee_role.role_arn)],
actions=["s3:ListBucket*"],
resources=[s3_accesspoint_arn],
conditions={"StringLike": {
"s3:prefix": f"{table_prefix}*"
}})
])
s3_accesspoint = s3.CfnAccessPoint(
self,
"S3AccessPoint",
bucket=f"{table_bucket}",
name=s3_accesspoint_name,
# network_origin = "Internet",
policy=s3_accesspoint_policy,
vpc_configuration=s3.CfnAccessPoint.VpcConfigurationProperty(
vpc_id=grantee_vpc.value_as_string))
glue_table_accesspoint_path_output = f"arn:aws:s3:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:accesspoint/{s3_accesspoint.name}/object/{table_prefix}"
# Output
core.CfnOutput(self,
"IAMRoleArnOutput",
value=grantee_role.role_arn,
description="IAM Role Arn")
core.CfnOutput(self,
"GlueTableOutput",
value=glue_table.table_arn,
description="Glue Table ARN")
core.CfnOutput(self,
"S3AccessPointPathOutput",
value=glue_table_accesspoint_path_output,
description="S3 Access Point Path for Glue Table")
def __init__(self, app: App, id: str) -> None:
super().__init__(app, id)
self.access_point = f"arn:aws:s3:{Aws.REGION}:{Aws.ACCOUNT_ID}:accesspoint/" \
f"{S3_ACCESS_POINT_NAME}"
# Set up a bucket
bucket = s3.Bucket(
self,
"example-bucket",
access_control=s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL,
encryption=s3.BucketEncryption.S3_MANAGED,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL)
# Delegating access control to access points
# https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-points-policies.html
bucket.add_to_resource_policy(
iam.PolicyStatement(
actions=["*"],
principals=[iam.AnyPrincipal()],
resources=[bucket.bucket_arn,
bucket.arn_for_objects('*')],
conditions={
"StringEquals": {
"s3:DataAccessPointAccount": f"{Aws.ACCOUNT_ID}"
}
}))
# lambda to process our objects during retrieval
retrieve_transformed_object_lambda = _lambda.Function(
self,
"retrieve_transformed_obj_lambda",
runtime=_lambda.Runtime.PYTHON_3_8,
handler="index.handler",
code=_lambda.Code.from_asset(
"lambda/retrieve_transformed_object_lambda"))
# Object lambda s3 access
retrieve_transformed_object_lambda.add_to_role_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=["*"],
actions=["s3-object-lambda:WriteGetObjectResponse"]))
# Restrict Lambda to be invoked from own account
retrieve_transformed_object_lambda.add_permission(
"invocationRestriction",
action="lambda:InvokeFunction",
principal=iam.AccountRootPrincipal(),
source_account=Aws.ACCOUNT_ID)
# Associate Bucket's access point with lambda get access
policy_doc = iam.PolicyDocument()
policy_statement = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["s3:GetObject"],
principals=[
iam.ArnPrincipal(
retrieve_transformed_object_lambda.role.role_arn)
],
resources=[f"{self.access_point}/object/*"])
policy_statement.sid = "AllowLambdaToUseAccessPoint"
policy_doc.add_statements(policy_statement)
example_bucket_ap = s3.CfnAccessPoint(self,
"example-bucket_ap",
bucket=bucket.bucket_name,
name=S3_ACCESS_POINT_NAME,
policy=policy_doc)
# Access point to receive GET request and use lambda to process objects
object_lambda_ap = s3_object_lambda.CfnAccessPoint(
self,
"s3_object_lambda_ap",
name=OBJECT_LAMBDA_ACCESS_POINT_NAME,
object_lambda_configuration=s3_object_lambda.CfnAccessPoint.
ObjectLambdaConfigurationProperty(
supporting_access_point=self.access_point,
transformation_configurations=[
s3_object_lambda.CfnAccessPoint.
TransformationConfigurationProperty(
actions=["GetObject"],
content_transformation={
"AwsLambda": {
"FunctionArn":
f"{retrieve_transformed_object_lambda.function_arn}"
}
})
]))
CfnOutput(self, "exampleBucketArn", value=bucket.bucket_arn)
CfnOutput(self,
"objectLambdaArn",
value=retrieve_transformed_object_lambda.function_arn)
CfnOutput(self,
"objectLambdaAccessPointArn",
value=object_lambda_ap.attr_arn)
CfnOutput(
self,
"objectLambdaAccessPointUrl",
value=f"https://console.aws.amazon.com/s3/olap/{Aws.ACCOUNT_ID}/"
f"{OBJECT_LAMBDA_ACCESS_POINT_NAME}?region={Aws.REGION}")