def __init__(
self,
scope: Construct,
id: str,
function_id: str,
bucket: S3Bucket,
config: BaseClusterConfig,
execution_role: iam.CfnRole,
handler_func: str,
timeout: int = 900,
):
super().__init__(scope, id)
function_name = f"pcluster-{function_id}-{self._stack_unique_id()}"
self.log_group = logs.CfnLogGroup(
scope,
f"{function_id}FunctionLogGroup",
log_group_name=f"/aws/lambda/{function_name}",
retention_in_days=get_cloud_watch_logs_retention_days(config),
)
self.log_group.cfn_options.deletion_policy = get_log_group_deletion_policy(
config)
self.lambda_func = awslambda.CfnFunction(
scope,
f"{function_id}Function",
function_name=function_name,
code=awslambda.CfnFunction.CodeProperty(
s3_bucket=bucket.name,
s3_key=
f"{bucket.artifact_directory}/custom_resources/artifacts.zip",
),
handler=f"{handler_func}.handler",
memory_size=128,
role=execution_role,
runtime="python3.8",
timeout=timeout,
)
def _add_lambda_cleanup(self, policy_statements, build_tags):
lambda_cleanup_execution_role = None
if self.custom_cleanup_lambda_role:
execution_role = self.custom_cleanup_lambda_role
else:
# LambdaCleanupPolicies
self._add_resource_delete_policy(
policy_statements,
["cloudformation:DeleteStack"],
[
self.format_arn(
service="cloudformation",
resource="stack",
resource_name="{0}/{1}".format(
self.image_id, self._stack_unique_id()),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["ec2:CreateTags"],
[
self.format_arn(
service="ec2",
account="",
resource="image",
region=region,
resource_name="*",
) for region in self._get_distribution_regions()
],
)
self._add_resource_delete_policy(
policy_statements,
["tag:TagResources"],
["*"],
)
self._add_resource_delete_policy(
policy_statements,
[
"iam:DetachRolePolicy", "iam:DeleteRole",
"iam:DeleteRolePolicy"
],
[
self.format_arn(
service="iam",
resource="role",
region="",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX + "Cleanup"),
),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["lambda:DeleteFunction", "lambda:RemovePermission"],
[
self.format_arn(
service="lambda",
resource="function",
sep=":",
resource_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["logs:DeleteLogGroup"],
[
self.format_arn(
service="logs",
resource="log-group",
sep=":",
resource_name="/aws/lambda/{0}:*".format(
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX)),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["iam:RemoveRoleFromInstanceProfile"],
[
self.format_arn(
service="iam",
resource="instance-profile",
region="",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
),
)
],
)
self._add_resource_delete_policy(
policy_statements,
["iam:DetachRolePolicy", "iam:DeleteRolePolicy"],
[
self.format_arn(
service="iam",
resource="role",
region="",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
),
)
],
)
self._add_resource_delete_policy(
policy_statements,
[
"SNS:GetTopicAttributes", "SNS:DeleteTopic",
"SNS:Unsubscribe"
],
[
self.format_arn(
service="sns",
resource="{0}".format(
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX)),
)
],
)
policy_document = iam.PolicyDocument(statements=policy_statements)
managed_lambda_policy = [
Fn.sub(
"arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
),
]
# LambdaCleanupExecutionRole
lambda_cleanup_execution_role = iam.CfnRole(
self,
"DeleteStackFunctionExecutionRole",
managed_policy_arns=managed_lambda_policy,
assume_role_policy_document=get_assume_role_policy_document(
"lambda.amazonaws.com"),
path=IAM_ROLE_PATH,
policies=[
iam.CfnRole.PolicyProperty(
policy_document=policy_document,
policy_name="LambdaCleanupPolicy",
),
],
tags=build_tags,
role_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX + "Cleanup"),
)
execution_role = lambda_cleanup_execution_role.attr_arn
# LambdaCleanupEnv
lambda_env = awslambda.CfnFunction.EnvironmentProperty(
variables={"IMAGE_STACK_ARN": self.stack_id})
# LambdaCWLogGroup
lambda_log = logs.CfnLogGroup(
self,
"DeleteStackFunctionLog",
log_group_name="/aws/lambda/{0}".format(
self._build_resource_name(IMAGEBUILDER_RESOURCE_NAME_PREFIX)),
)
# LambdaCleanupFunction
lambda_cleanup = awslambda.CfnFunction(
self,
"DeleteStackFunction",
function_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
code=awslambda.CfnFunction.CodeProperty(
s3_bucket=self.config.custom_s3_bucket
or S3Bucket.get_bucket_name(
AWSApi.instance().sts.get_account_id(), get_region()),
s3_key=self.bucket.get_object_key(S3FileType.CUSTOM_RESOURCES,
"artifacts.zip"),
),
handler="delete_image_stack.handler",
memory_size=128,
role=execution_role,
runtime="python3.8",
timeout=900,
environment=lambda_env,
tags=build_tags,
)
permission = awslambda.CfnPermission(
self,
"DeleteStackFunctionPermission",
action="lambda:InvokeFunction",
principal="sns.amazonaws.com",
function_name=lambda_cleanup.attr_arn,
source_arn=Fn.ref("BuildNotificationTopic"),
)
lambda_cleanup.add_depends_on(lambda_log)
return lambda_cleanup, permission, lambda_cleanup_execution_role, lambda_log
def __init__(self, scope: core.Construct, id: str, data, iam_vars) -> None:
super().__init__(scope, id)
# VPC
vpc = ec2.CfnVPC(self, "cdk-vpc", cidr_block=data["vpc"])
igw = ec2.CfnInternetGateway(self, id="igw")
ec2.CfnVPCGatewayAttachment(self,
id="igw-attach",
vpc_id=vpc.ref,
internet_gateway_id=igw.ref)
public_route_table = ec2.CfnRouteTable(self,
id="public_route_table",
vpc_id=vpc.ref)
ec2.CfnRoute(self,
id="public_route",
route_table_id=public_route_table.ref,
destination_cidr_block="0.0.0.0/0",
gateway_id=igw.ref)
public_subnets = []
for i, s in enumerate(data["subnets"]["public"]):
subnet = ec2.CfnSubnet(self,
id="public_{}".format(s),
cidr_block=s,
vpc_id=vpc.ref,
availability_zone=core.Fn.select(
i, core.Fn.get_azs()),
map_public_ip_on_launch=True)
public_subnets.append(subnet)
ec2.CfnSubnetRouteTableAssociation(
self,
id="public_{}_association".format(s),
route_table_id=public_route_table.ref,
subnet_id=subnet.ref)
eip = ec2.CfnEIP(self, id="natip")
nat = ec2.CfnNatGateway(self,
id="nat",
allocation_id=eip.attr_allocation_id,
subnet_id=public_subnets[0].ref)
private_route_table = ec2.CfnRouteTable(self,
id="private_route_table",
vpc_id=vpc.ref)
ec2.CfnRoute(self,
id="private_route",
route_table_id=private_route_table.ref,
destination_cidr_block="0.0.0.0/0",
nat_gateway_id=nat.ref)
private_subnets = []
for i, s in enumerate(data["subnets"]["private"]):
subnet = ec2.CfnSubnet(self,
id="private_{}".format(s),
cidr_block=s,
vpc_id=vpc.ref,
availability_zone=core.Fn.select(
i, core.Fn.get_azs()),
map_public_ip_on_launch=False)
private_subnets.append(subnet)
ec2.CfnSubnetRouteTableAssociation(
self,
id="private_{}_association".format(s),
route_table_id=private_route_table.ref,
subnet_id=subnet.ref)
# Security groups
lb_sg = ec2.CfnSecurityGroup(self,
id="lb",
group_description="LB SG",
vpc_id=vpc.ref)
lambda_sg = ec2.CfnSecurityGroup(self,
id="lambda",
group_description="Lambda SG",
vpc_id=vpc.ref)
public_prefix = ec2.CfnPrefixList(self,
id="cidr_prefix",
address_family="IPv4",
max_entries=1,
prefix_list_name="public",
entries=[{
"cidr": "0.0.0.0/0",
"description": "Public"
}])
_sg_rules = [{
'sg':
lb_sg.attr_group_id,
'rules': [{
"direction": "ingress",
"description": "HTTP from Internet",
"from_port": 80,
"to_port": 80,
"protocol": "tcp",
"cidr_blocks": public_prefix.ref
}, {
"direction": "egress",
"description": "LB to Lambda",
"from_port": 80,
"to_port": 80,
"protocol": "tcp",
"source_security_group_id": lambda_sg.attr_group_id
}]
}, {
"sg":
lambda_sg.attr_group_id,
"rules": [{
"direction": "ingress",
"description": "HTTP from LB",
"from_port": 80,
"to_port": 80,
"protocol": "tcp",
"source_security_group_id": lb_sg.attr_group_id
}, {
"direction": "egress",
"description": "All to Internet",
"from_port": 0,
"to_port": 65535,
"protocol": "tcp",
"cidr_blocks": public_prefix.ref
}]
}]
for ruleset in _sg_rules:
for rule in ruleset["rules"]:
if rule["direction"] == "ingress":
ec2.CfnSecurityGroupIngress(
self,
id=rule["description"].replace(" ", "_"),
description=rule["description"],
to_port=rule["to_port"],
from_port=rule["from_port"],
ip_protocol=rule["protocol"],
group_id=ruleset["sg"],
source_prefix_list_id=rule["cidr_blocks"]
if "cidr_blocks" in rule else None,
source_security_group_id=rule[
"source_security_group_id"]
if "source_security_group_id" in rule else None)
else:
ec2.CfnSecurityGroupEgress(
self,
id=rule["description"].replace(" ", "_"),
description=rule["description"],
to_port=rule["to_port"],
from_port=rule["from_port"],
ip_protocol=rule["protocol"],
group_id=ruleset["sg"],
destination_prefix_list_id=rule["cidr_blocks"]
if "cidr_blocks" in rule else None,
destination_security_group_id=rule[
"source_security_group_id"]
if "source_security_group_id" in rule else None)
# IAM
assume_policy_doc = iam.PolicyDocument()
for statement in iam_vars["assume"]["Statement"]:
_statement = iam.PolicyStatement(actions=[statement["Action"]], )
_statement.add_service_principal(statement["Principal"]["Service"])
assume_policy_doc.add_statements(_statement)
role = iam.CfnRole(self,
id="iam_role",
path="/",
assume_role_policy_document=assume_policy_doc)
role_policy_doc = iam.PolicyDocument()
for statement in iam_vars["policy"]["Statement"]:
_statement = iam.PolicyStatement(actions=statement["Action"],
resources=["*"])
role_policy_doc.add_statements(_statement)
policy = iam.CfnPolicy(self,
id="iam_policy",
policy_document=role_policy_doc,
policy_name="cdkPolicy",
roles=[role.ref])
# Lambda
shutil.make_archive("../lambda", 'zip', "../lambda/")
s3_client = boto3.client('s3')
s3_client.upload_file("../lambda.zip", "cloudevescops-zdays-demo",
"cdk.zip")
function = lmd.CfnFunction(self,
id="lambda_function",
handler="lambda.lambda_handler",
role=role.attr_arn,
runtime="python3.7",
code={
"s3Bucket": "cloudevescops-zdays-demo",
"s3Key": "cdk.zip"
},
vpc_config={
"securityGroupIds": [lambda_sg.ref],
"subnetIds":
[s.ref for s in private_subnets]
},
environment={"variables": {
"TOOL": "CDK"
}})
# LB
lb = alb.CfnLoadBalancer(self,
id="alb",
name="lb-cdk",
scheme="internet-facing",
type="application",
subnets=[s.ref for s in public_subnets],
security_groups=[lb_sg.ref])
lmd.CfnPermission(self,
id="lambda_permis",
action="lambda:InvokeFunction",
function_name=function.ref,
principal="elasticloadbalancing.amazonaws.com")
tg = alb.CfnTargetGroup(self,
id="alb_tg",
name="lambda-cdk",
target_type="lambda",
health_check_enabled=True,
health_check_interval_seconds=40,
health_check_path="/",
health_check_timeout_seconds=30,
targets=[{
"id":
function.get_att("Arn").to_string()
}],
matcher={"httpCode": "200"})
alb.CfnListener(self,
id="listener",
default_actions=[{
"type": "forward",
"targetGroupArn": tg.ref
}],
load_balancer_arn=lb.ref,
port=80,
protocol="HTTP")
core.CfnOutput(self, id="fqdn", value=lb.attr_dns_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Exercise 9
# LambdaExecutionRole
lambda_execution_role = iam.CfnRole(
self,
"LambdaExecutionRole",
role_name="lambda-execution-role",
assume_role_policy_document={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
path="/",
policies=[{
"policyName": "root",
"policyDocument": {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": [
"logs:CreateLogGroup", "logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource":
"arn:aws:logs:*:*:*"
}, {
"Effect": "Allow",
"Action": ["ec2:Describe*"],
"Resource": "*"
}]
}
}])
# EdxProjectCloud9
edx_project_cloud9 = cloud9.CfnEnvironmentEC2(
self,
"EdxProjectCloud9",
automatic_stop_time_minutes=30,
instance_type="t2.micro",
name="BuildingOnAWS%s" % (core.Aws.STACK_NAME),
subnet_id=core.Fn.import_value("PublicSubnet1"))
edx_project_cloud9.apply_removal_policy(core.RemovalPolicy.DESTROY)
# CustomFunction
custom_function = lambda_.CfnFunction(
self,
"CustomFunction",
runtime="nodejs8.10",
handler="index.handler",
code={
"zipFile":
"const response = require('cfn-response');\n"
"const AWS = require('aws-sdk');\n"
"exports.handler = (event, context) => {\n"
"let params = {\n"
"Filters: [\n"
"{\n"
"Name: 'tag:aws:cloud9:environment',\n"
"Values: [\n"
"event.ResourceProperties.EdxProjectCloud9\n"
"]\n"
"}\n"
"]\n"
"};\n"
"let ec2 = new AWS.EC2();\n"
"ec2.describeInstances(params, (err, data) => {\n"
"if (err) {\n"
"console.log(err, err.stack); // an error occurred\n"
"response.send(event, context, response.FAILED, {});\n"
"}else{\n"
"let responseData = {Value: data.Reservations[0].Instances[0].SecurityGroups[0].GroupId};\n"
"console.log(responseData);\n"
"response.send(event, context, response.SUCCESS, responseData);\n"
"}\n"
"});\n"
"};"
},
timeout=30,
role=lambda_execution_role.attr_arn)
# CustomResource
custom_resource = cfn.CfnCustomResource(
self,
"CustomResource",
service_token=custom_function.attr_arn,
)
custom_resource.add_override("Properties.EdxProjectCloud9",
edx_project_cloud9.ref)
# Output
core.CfnOutput(self,
"EdxProjectCloud9Output",
value=edx_project_cloud9.ref,
description="Edx User Cloud9",
export_name="EdxProjectCloud9")
core.CfnOutput(self,
"EdxProjectCloud9SgOutput",
value=core.Fn.get_att(custom_resource.logical_id,
"Value").to_string(),
description="Edx User Cloud9 Security Group ID",
export_name="EdxProjectCloud9Sg")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Set Parameters
db_password_parameters = core.CfnParameter(
self,
"DBPassword",
no_echo=True,
description="New account and RDS password",
min_length=1,
max_length=41,
constraint_description=
"the password must be between 1 and 41 characters",
default="DBPassword")
# LambdaExecutionRole
LambdaExecutionRole = iam.CfnRole(
self,
"LabelsLambdaExecutionRole",
assume_role_policy_document={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
managed_policy_arns=[
"arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole",
"arn:aws:iam::aws:policy/AmazonRekognitionReadOnlyAccess",
"arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess",
"arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess"
],
policies=[{
"policyName": "root",
"policyDocument": {
"Version":
"2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*"
},
]
}
}])
# S3 Bucket
source_bucket = "sourcebucketname%s" % (core.Aws.ACCOUNT_ID)
# LabelsLambda
LabelsLambda = lambda_.CfnFunction(
self,
"LabelsLambda",
handler="lambda_function.lambda_handler",
role=LambdaExecutionRole.attr_arn,
code={
"s3Bucket": source_bucket,
"s3Key": "lambda.zip"
},
runtime="python3.6",
timeout=120,
tracing_config={"mode": "Active"},
vpc_config={
"securityGroupIds":
[core.Fn.import_value("LambdaSecurityGroupOutput")],
"subnetIds": [
core.Fn.import_value("PrivateSubnet1"),
core.Fn.import_value("PrivateSubnet2")
]
},
environment={
"variables": {
"DATABASE_HOST": core.Fn.import_value("MyDBEndpoint"),
"DATABASE_USER": "******",
"DATABASE_PASSWORD":
db_password_parameters.value_as_string,
"DATABASE_DB_NAME": "Photos"
}
})
# UploadQueue
upload_queue = sqs.CfnQueue(self,
"UploadQueue",
queue_name="uploads-queue",
message_retention_period=12800,
visibility_timeout=300)
# UploadSNSTopic
upload_sns_topic = sns.CfnTopic(
self,
"UploadSNSTopic",
display_name="uploads-topic",
subscription=[{
"endpoint": upload_queue.attr_arn,
"protocol": "sqs"
}, {
"endpoint": LabelsLambda.attr_arn,
"protocol": "lambda"
}],
)
# QueuePolicy
queue_policy = sqs.CfnQueuePolicy(
self,
"QueuePolicy",
queues=[upload_queue.ref],
policy_document={
"Version":
"2012-10-17",
"Id":
"QueuePolicy",
"Statement": [{
"Sid": "Allow-SendMessage-To-Queues-From-SNS-Topic",
"Effect": "Allow",
"Principal": "*",
"Action": ["SQS:SendMessage"],
"Resource": "*",
"Condition": {
"ArnEquals": {
"aws:SourceArn": "%s" % (upload_sns_topic.ref)
}
}
}]
})
# UploadTopicPolicy
upload_topic_policy = sns.CfnTopicPolicy(
self,
"UploadTopicPolicy",
policy_document={
"Version":
"2012-10-17",
"Id":
"QueuePolicy",
"Statement": [{
"Sid": "Allow-S3-Publish",
"Effect": "Allow",
"Principal": {
"Service": "s3.amazonaws.com"
},
"Action": ["SNS:Publish"],
"Resource": upload_sns_topic.ref,
"Condition": {
"StringEquals": {
"aws:SourceAccount": "!Sub '${AWS::AccountId}'"
},
"ArnLike": {
"aws:SourceArn": {
"Fn::Join": [
"",
[
"arn:aws:s3:*:*:",
"!Sub 'imagebucketsns${AWS::AccountId}'"
]
]
}
}
},
}]
},
topics=[upload_sns_topic.ref])
# ImageS3Bucket
image_s3_bucket = s3.CfnBucket(self,
"ImageS3Bucket",
bucket_name="imagebucketsns%s" %
(core.Aws.ACCOUNT_ID),
notification_configuration={
"topicConfigurations": [{
"event":
's3:ObjectCreated:*',
"topic":
upload_sns_topic.ref
}]
})
image_s3_bucket.add_depends_on(upload_topic_policy)
image_s3_bucket.apply_removal_policy(core.RemovalPolicy.DESTROY)
# ImageS3BucketPermission
ImageS3BucketPermission = lambda_.CfnPermission(
self,
"ImageS3BucketPermission",
action="lambda:InvokeFunction",
function_name=LabelsLambda.attr_arn,
principal="sns.amazonaws.com",
source_arn=upload_sns_topic.ref)
# Outputs
core.CfnOutput(self,
"ImageS3BucketOutput",
value=image_s3_bucket.ref,
description="Image S3 Bucket",
export_name="ImageS3Bucket")
core.CfnOutput(self,
"LabelsLambdaOutput",
value=LabelsLambda.ref,
description="Labels Lambda",
export_name="LabelsLambda")