def create_endpoints(self) -> None:
endpoints = {
"ECS": ec2.InterfaceVpcEndpointAwsService.ECS,
"ECR": ec2.InterfaceVpcEndpointAwsService.ECR,
"ECR_DOCKER": ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER,
"CLOUDWATCH_LOGS":
ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS,
"SECRETS_MANAGER":
ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER,
}
for name, service in endpoints.items():
ec2.InterfaceVpcEndpoint(
self,
name,
vpc=self.instance,
service=service,
subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED),
private_dns_enabled=True,
security_groups=[self.vpc_endpoint_sg],
)
self.instance.add_gateway_endpoint(
"s3-endpoint",
service=ec2.GatewayVpcEndpointAwsService.S3,
subnets=[ec2.SubnetSelection(subnet_type=ec2.SubnetType.ISOLATED)],
)
def __init__(self, scope: core.Construct, id: str, vpc_id: str, az_cidr: dict, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self._subnets = []
stack_id = id
for az in az_cidr.keys():
cidr_per_zone=az_cidr.get(az)
for cidr in cidr_per_zone:
subnet = ec2.Subnet(
self,
id=stack_id+"-"+az,
availability_zone=az,
cidr_block=cidr,
vpc_id=vpc_id,
map_public_ip_on_launch=False
)
self.add(subnet)
vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id=vpc_id)
selected_subnets = ec2.SubnetSelection(subnets=self._subnets)
ec2.InterfaceVpcEndpoint(self, stack_id+"ec2",
vpc=vpc,
service=ec2.InterfaceVpcEndpointAwsService.E_C2,
subnets=selected_subnets
)
ec2.InterfaceVpcEndpoint(self, stack_id+"ecr",
vpc=vpc,
service=ec2.InterfaceVpcEndpointAwsService.ECR,
subnets=selected_subnets
)
ec2.InterfaceVpcEndpoint(self, stack_id+"drk",
vpc=vpc,
service=ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER,
subnets=selected_subnets
)
ec2.GatewayVpcEndpoint(self, stack_id+"s3",
vpc=vpc,
service=ec2.GatewayVpcEndpointAwsService.S3,
subnets=[selected_subnets]
)
def add_interfaces(self, services:List[str]):
for svc in services:
if not svc in self.interfaces:
self.interfaces[svc] = ec2.InterfaceVpcEndpoint(
self, svc,
vpc=self.vpc,
service=ec2.InterfaceVpcEndpointAwsService(name=svc),
open=True,
private_dns_enabled=True,
lookup_supported_azs=True,
security_groups=[self.security_group])
return self
def vpc_creation(self, name_extension, conf, stage):
resource_name = name_extension + "-vpc.main"
self.vpc = _ec2.Vpc(
self,
resource_name,
cidr=conf[stage]["vpc_cidr"],
max_azs=conf[stage]["vpc_az"],
nat_gateway_provider=_ec2.NatProvider.gateway(),
nat_gateways=conf[stage]["nat_gateways_num"],
subnet_configuration=[
_ec2.SubnetConfiguration(
subnet_type=_ec2.SubnetType.PUBLIC,
name="Public",
cidr_mask=conf[stage]["subnet_cidr_mask"]),
_ec2.SubnetConfiguration(
subnet_type=_ec2.SubnetType.PRIVATE,
name="Private",
cidr_mask=conf[stage]["subnet_cidr_mask"]),
],
)
endpoints_list = [
("logs", _ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS),
("ecr", _ec2.InterfaceVpcEndpointAwsService.ECR),
("ecr_dk", _ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER),
]
for ep_str, svc in endpoints_list:
ep = _ec2.InterfaceVpcEndpoint(
self,
ep_str,
vpc=self.vpc,
service=svc,
subnets={"subnet_type": _ec2.SubnetType.PRIVATE})
self.objects_list.append(ep)
self.s3_ep = _ec2.GatewayVpcEndpoint(
self,
"s3",
vpc=self.vpc,
service=_ec2.GatewayVpcEndpointAwsService.S3,
subnets=[
_ec2.SubnetSelection(subnet_type=_ec2.SubnetType.PRIVATE)
])
self.objects_list.append(self.s3_ep)
self.objects_list.append(self.vpc)
core.Tag.add(self.vpc, "Name", conf[stage]["vpc_name"])
def create_interface_endpoint(
self,
service_name: str,
security_group: _ec2.ISecurityGroup,
vpc: _ec2.Vpc,
interface_endpoint_policy: _iam.PolicyStatement = None):
"""create interface endpoint"""
vpc_endpoint = _ec2.InterfaceVpcEndpoint(
self,
id=service_name.upper() + "VPCEndPoint",
vpc=vpc,
service=_ec2.InterfaceVpcEndpointAwsService(service_name),
private_dns_enabled=True,
security_groups=[security_group])
if interface_endpoint_policy is not None:
vpc_endpoint.add_to_policy(interface_endpoint_policy)
def __init__(self, scope: core.Construct, id: str, vpc: ec2.IVpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.security_group = ec2.SecurityGroup(
self,
'EndpointSecurity',
vpc=vpc,
allow_all_outbound=True,
description='SG for AWS Resources in isolated subnet')
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.ALL,
string_representation='Any source'))
self.gateways = {}
for svc in ['s3', 'dynamodb']:
self.gateways[svc] = ec2.GatewayVpcEndpoint(
self,
svc,
vpc=vpc,
service=ec2.GatewayVpcEndpointAwsService(name=svc))
self.interfaces = {}
for svc in [
'ssm', 'ec2messages', 'ec2', 'ssmmessages', 'kms',
'elasticloadbalancing', 'elasticfilesystem', 'lambda',
'states', 'events', 'execute-api', 'kinesis-streams',
'kinesis-firehose', 'logs', 'sns', 'sqs', 'secretsmanager',
'config', 'ecr.api', 'ecr.dkr'
]:
self.interfaces[svc] = ec2.InterfaceVpcEndpoint(
self,
svc,
vpc=vpc,
service=ec2.InterfaceVpcEndpointAwsService(name=svc),
open=True,
private_dns_enabled=True,
lookup_supported_azs=False,
security_groups=[self.security_group])
def create_endpoints(self) -> None:
endpoints = {
"SQS":
ec2.InterfaceVpcEndpointAwsService.SQS,
"CLOUDWATCH_LOGS":
ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS,
"CLOUDWATCH_MONITORING":
ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH,
"KMS ":
ec2.InterfaceVpcEndpointAwsService.KMS,
"ECR":
ec2.InterfaceVpcEndpointAwsService.ECR,
"ECR_DOCKER":
ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER,
"CODE_ARTIFACT_API":
ec2.InterfaceVpcEndpointAwsService(name="codeartifact.api"),
"CODE_ARTIFACT_REPOSITORIES":
ec2.InterfaceVpcEndpointAwsService(
name="codeartifact.repositories"),
}
for name, service in endpoints.items():
ec2.InterfaceVpcEndpoint(
self,
name,
vpc=self.instance,
service=service,
subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED),
private_dns_enabled=True,
security_groups=[self.mwaa_sg],
)
self.instance.add_gateway_endpoint(
"s3-endpoint",
service=ec2.GatewayVpcEndpointAwsService.S3,
subnets=[ec2.SubnetSelection(subnet_type=ec2.SubnetType.ISOLATED)],
)
def provision_vpc(self, name: str, vpc: VPC):
self.public_subnet_name = f"{name}-public"
self.private_subnet_name = f"{name}-private"
if not vpc.create:
self.vpc = ec2.Vpc.from_lookup("Vpc", vpc_id=vpc.id)
return
nat_provider = ec2.NatProvider.gateway()
self.vpc = ec2.Vpc(
self.scope,
"VPC",
max_azs=vpc.max_azs,
cidr=vpc.cidr,
subnet_configuration=[
ec2.SubnetConfiguration(
subnet_type=ec2.SubnetType.PUBLIC,
name=self.public_subnet_name,
cidr_mask=24, # can't use token ids
),
ec2.SubnetConfiguration(
subnet_type=ec2.SubnetType.PRIVATE,
name=self.private_subnet_name,
cidr_mask=24, # can't use token ids
),
],
gateway_endpoints={
"S3":
ec2.GatewayVpcEndpointOptions(
service=ec2.GatewayVpcEndpointAwsService.S3),
},
nat_gateway_provider=nat_provider,
)
cdk.Tags.of(self.vpc).add("Name", name)
cdk.CfnOutput(self.scope, "vpc-output", value=self.vpc.vpc_cidr_block)
# ripped off this: https://github.com/aws/aws-cdk/issues/9573
pod_cidr = ec2.CfnVPCCidrBlock(self.scope,
"PodCidr",
vpc_id=self.vpc.vpc_id,
cidr_block="100.64.0.0/16")
c = 0
for az in self.vpc.availability_zones:
pod_subnet = ec2.PrivateSubnet(
self.scope,
# this can't be okay
f"{name}-pod-{c}", # Can't use parameter/token in this name
vpc_id=self.vpc.vpc_id,
availability_zone=az,
cidr_block=f"100.64.{c}.0/18",
)
pod_subnet.add_default_nat_route([
gw for gw in nat_provider.configured_gateways if gw.az == az
][0].gateway_id)
pod_subnet.node.add_dependency(pod_cidr)
# TODO: need to tag
c += 64
for endpoint in [
"ec2", # Only these first three have predefined consts
"sts",
"ecr.api",
"autoscaling",
"ecr.dkr",
]: # TODO: Do we need an s3 interface as well? or just the gateway?
self.vpc_endpoint = ec2.InterfaceVpcEndpoint(
self.scope,
f"{endpoint}-ENDPOINT",
vpc=self.vpc,
service=ec2.InterfaceVpcEndpointAwsService(endpoint, port=443),
# private_dns_enabled=True,
subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE),
)
def __init__(
self,
scope: core.Construct,
id: str,
vpc,
stack_log_level: str,
back_end_api_name: str,
**kwargs
) -> None:
super().__init__(scope, id, **kwargs)
# Create Serverless Event Processor using Lambda):
# Read Lambda Code):
try:
with open("secure_private_api/stacks/back_end/lambda_src/serverless_greeter.py", mode="r") as f:
greeter_fn_code = f.read()
except OSError as e:
print("Unable to read Lambda Function Code")
raise e
greeter_fn = _lambda.Function(
self,
"getSquareFn",
function_name=f"greeter_fn_{id}",
runtime=_lambda.Runtime.PYTHON_3_7,
handler="index.lambda_handler",
code=_lambda.InlineCode(greeter_fn_code),
timeout=core.Duration.seconds(15),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": f"{stack_log_level}",
"Environment": "Production",
"ANDON_CORD_PULLED": "False"
}
)
greeter_fn_version = greeter_fn.latest_version
greeter_fn_version_alias = _lambda.Alias(
self,
"greeterFnAlias",
alias_name="MystiqueAutomation",
version=greeter_fn_version
)
# Create Custom Loggroup
# /aws/lambda/function-name
greeter_fn_lg = _logs.LogGroup(
self,
"squareFnLoggroup",
log_group_name=f"/aws/lambda/{greeter_fn.function_name}",
retention=_logs.RetentionDays.ONE_WEEK,
removal_policy=core.RemovalPolicy.DESTROY
)
# Add API GW front end for the Lambda
back_end_01_api_stage_options = _apigw.StageOptions(
stage_name="miztiik",
throttling_rate_limit=10,
throttling_burst_limit=100,
logging_level=_apigw.MethodLoggingLevel.INFO
)
# Lets create a private secure end point
# Create a security group dedicated to our API Endpoint
self.secure_private_api_01_sec_grp = _ec2.SecurityGroup(
self,
"secureApi01SecurityGroup",
vpc=vpc,
allow_all_outbound=True,
description="Miztiik Automation: Secure our private API using security groups"
)
# Allow 443 inbound on our Security Group
self.secure_private_api_01_sec_grp.add_ingress_rule(
_ec2.Peer.ipv4(vpc.vpc_cidr_block),
_ec2.Port.tcp(443)
)
secure_private_api_01_endpoint = _ec2.InterfaceVpcEndpoint(
self,
"secureApi01Endpoint",
vpc=vpc,
service=_ec2.InterfaceVpcEndpointAwsService.APIGATEWAY,
private_dns_enabled=True,
subnets=_ec2.SubnetSelection(
subnet_type=_ec2.SubnetType.ISOLATED
)
)
# Create a API Gateway Resource Policy to attach to API GW
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-restapi.html#cfn-apigateway-restapi-policy
secure_private_api_01_res_policy = _iam.PolicyDocument(
statements=[
_iam.PolicyStatement(
principals=[_iam.AnyPrincipal()],
actions=["execute-api:Invoke"],
# resources=[f"{api_01.arn_for_execute_api(method="GET",path="greeter", stage="miztiik")}"],
resources=[core.Fn.join("", ["execute-api:/", "*"])],
effect=_iam.Effect.DENY,
conditions={
"StringNotEquals":
{
"aws:sourceVpc": f"{secure_private_api_01_endpoint.vpc_endpoint_id}"
}
},
sid="DenyAllNonVPCAccessToApi"
),
_iam.PolicyStatement(
principals=[_iam.AnyPrincipal()],
actions=["execute-api:Invoke"],
resources=[core.Fn.join("", ["execute-api:/", "*"])],
effect=_iam.Effect.ALLOW,
sid="AllowVPCAccessToApi"
)
]
)
# Create API Gateway
secure_private_api_01 = _apigw.RestApi(
self,
"backEnd01Api",
rest_api_name=f"{back_end_api_name}",
deploy_options=back_end_01_api_stage_options,
endpoint_types=[
_apigw.EndpointType.PRIVATE
],
policy=secure_private_api_01_res_policy,
)
back_end_01_api_res = secure_private_api_01.root.add_resource("secure")
greeter = back_end_01_api_res.add_resource("greeter")
greeter_method_get = greeter.add_method(
http_method="GET",
request_parameters={
"method.request.header.InvocationType": True,
"method.request.path.number": True
},
integration=_apigw.LambdaIntegration(
handler=greeter_fn,
proxy=True
)
)
# Outputs
output_1 = core.CfnOutput(
self,
"SecureApiUrl",
value=f"{greeter.url}",
description="Use an utility like curl from the same VPC as the API to invoke it."
)
def __init__(self, scope: core.Construct, id: str, cidr_range: str,
tgw_asn: int, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# VPC Creation
self.vpc = ec2.Vpc(
self,
f"{kwargs['env']['region']}-vpc",
max_azs=1,
cidr=cidr_range,
# configuration will create 1 subnet in a single AZ.
subnet_configuration=[
ec2.SubnetConfiguration(subnet_type=ec2.SubnetType.ISOLATED,
name="Isolated",
cidr_mask=25)
])
# Transit Gateway creation
self.tgw = ec2.CfnTransitGateway(
self,
id=f"TGW-{kwargs['env']['region']}",
amazon_side_asn=tgw_asn,
auto_accept_shared_attachments="enable",
default_route_table_association="enable",
default_route_table_propagation="enable",
tags=[
core.CfnTag(key='Name', value=f"tgw-{kwargs['env']['region']}")
])
# Transit Gateway attachment to the VPC
self.tgw_attachment = ec2.CfnTransitGatewayAttachment(
self,
id=f"tgw-vpc-{kwargs['env']['region']}",
transit_gateway_id=self.tgw.ref,
vpc_id=self.vpc.vpc_id,
subnet_ids=[
subnet.subnet_id for subnet in self.vpc.isolated_subnets
],
tags=[
core.CfnTag(key='Name',
value=f"tgw-{self.vpc.vpc_id}-attachment")
])
# VPC Endpoint creation for SSM (3 Endpoints needed)
ec2.InterfaceVpcEndpoint(
self,
"VPCe - SSM",
service=ec2.InterfaceVpcEndpointService(
core.Fn.sub("com.amazonaws.${AWS::Region}.ssm")),
private_dns_enabled=True,
vpc=self.vpc,
)
ec2.InterfaceVpcEndpoint(
self,
"VPCe - EC2 Messages",
service=ec2.InterfaceVpcEndpointService(
core.Fn.sub("com.amazonaws.${AWS::Region}.ec2messages")),
private_dns_enabled=True,
vpc=self.vpc,
)
ec2.InterfaceVpcEndpoint(
self,
"VPCe - SSM Messages",
service=ec2.InterfaceVpcEndpointService(
core.Fn.sub("com.amazonaws.${AWS::Region}.ssmmessages")),
private_dns_enabled=True,
vpc=self.vpc,
)
def __init__(self, scope: cdk.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Create a VPC (2 AZ, each with private + public subnets and all the required components)
vpc = ec2.Vpc(self, "Target_VPC", cidr="10.0.0.0/16")
# Sec Grp
sg = ec2.SecurityGroup(self,
'SecurityGroup',
vpc=vpc,
allow_all_outbound=True,
security_group_name="VpcEndpoint_sg")
sg.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(443))
#VPC endpoit
vpc_endpoint = ec2.InterfaceVpcEndpoint(
self,
'ApiVpcEndpoint',
vpc=vpc,
private_dns_enabled=True,
security_groups=[sg],
service=ec2.InterfaceVpcEndpointAwsService(name="execute-api",
port=443))
handler = lambda_.Function(
self,
"sayHi_fn",
runtime=lambda_.Runtime.NODEJS_14_X,
code=lambda_.Code.from_asset("resources/lambda"),
handler="app.main")
"""
api = apigateway.RestApi(self, "sayHi_api",
rest_api_name="Say Hi API",
description="This service greets"
)
integration = apigateway.LambdaIntegration(handler,
request_templates={"application/json": '{ "statusCode": "200" }'}
)
api.root.add_method("GET", integration)
"""
apigateway.LambdaRestApi(
self,
"sayHiPrivateLambdaRestAPI",
endpoint_types=[apigateway.EndpointType.PRIVATE],
handler=handler,
policy=iam.PolicyDocument(statements=[
iam.PolicyStatement(principals=[iam.AnyPrincipal()],
actions=['execute-api:Invoke'],
resources=['execute-api:/*'],
effect=iam.Effect.DENY,
conditions={
"StringNotEquals": {
"aws:SourceVpce":
vpc_endpoint.vpc_endpoint_id
}
}),
iam.PolicyStatement(principals=[iam.AnyPrincipal()],
actions=['execute-api:Invoke'],
resources=['execute-api:/*'],
effect=iam.Effect.ALLOW)
]))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create a new ECR Repository
repo = ecr.Repository(
self, 'Repository'
)
# Reference the existing VPC with the name EnvironmentStack/VPC
vpc = ec2.Vpc.from_lookup(self, 'VPC', vpc_name='EnvironmentStack/VPC')
jsonPolicyDocument = {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ecr:*",
"cloudtrail:LookupEvents"
],
"Resource": repo.repository_arn,
"Condition": {
"StringEquals": {
"aws:SourceVpc": vpc.vpc_id
}
}
},
{
"Effect": "Allow",
"Action": "ecr:GetAuthorizationToken",
"Resource": "*",
"Condition": {
"StringEquals": {
"aws:SourceVpc": vpc.vpc_id
}
}
}
]
}
policyDocument=iam.PolicyDocument.from_json(jsonPolicyDocument)
# Create an IAM Role that has full access to this repository - but only from this VPC
ecrAccessRole = iam.Role(
self, "ECRAccessRole",
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal("codebuild.amazonaws.com"),
iam.ServicePrincipal("ec2.amazonaws.com")
),
inline_policies=[policyDocument]
)
instance_profile = iam.CfnInstanceProfile(
self, "InstanceProfile",
roles=[ecrAccessRole.role_name]
)
# Create a security group for our endpoints
security_group = ec2.SecurityGroup(
self, "ECR-SG",
vpc=vpc,
allow_all_outbound=True
)
# Allow 443 inbound on our Security Group
security_group.add_ingress_rule(
ec2.Peer.ipv4(vpc.vpc_cidr_block),
ec2.Port.tcp(443)
)
# Create VPC Endpoint for ECR API
ecrEndpoint = ec2.InterfaceVpcEndpoint(
self, 'ecr',
service=ec2.InterfaceVpcEndpointAwsService.ECR,
private_dns_enabled=True,
vpc=vpc,
security_groups=[security_group],
subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE
)
)
# Create VPC Endpoint for ECR Docker
ecrEndpointDocker = ec2.InterfaceVpcEndpoint(
self, 'ecrdkr',
service=ec2.InterfaceVpcEndpointAwsService.ECR_DOCKER,
private_dns_enabled=True,
vpc=vpc,
security_groups=[security_group],
subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE
)
)
# Create Gateway Endpoint for S3
s3Endpoint = ec2.GatewayVpcEndpoint(
self, 's3',
service=ec2.GatewayVpcEndpointAwsService.S3,
vpc=vpc,
subnets=[ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE
)]
)
def __init__(self,
scope: core.Construct,
id: str,
vpc: _ec2.Vpc,
redshift_secret_arn: str,
lambda_sg: _ec2.SecurityGroup,
clean_glue_db: _glue.Database,
redshift_role_arn: str,
redshift_cluster_endpoint: _redshift.Endpoint,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.__vpc = vpc
self.__redshift_secret_arn = redshift_secret_arn
self.__lambda_sg = lambda_sg
self.__clean_glue_db = clean_glue_db
self.__redshift_role_arn = redshift_role_arn
stack = Stack.of(self)
# Generate secrets for Redshift users
generator = SecretStringGenerator(exclude_characters="'", exclude_punctuation=True)
self.__etl_user_secret = Secret(
scope=self, id='ETLUserSecret',
description="ETL user Redshift",
generate_secret_string=SecretStringGenerator(
exclude_characters="'",
exclude_punctuation=True,
generate_string_key="password",
secret_string_template=json.dumps(
{
'username': _config.Redshift.ETL_USER,
'dbname': _config.Redshift.DATABASE,
'host': redshift_cluster_endpoint.hostname,
'port': core.Token.as_string(redshift_cluster_endpoint.port)
}
)
)
)
self.__dataengineer_user_secret = Secret(
scope=self, id='DataEngineerUserSecret',
description="DataEngineer user Redshift",
generate_secret_string=SecretStringGenerator(
exclude_characters="'",
exclude_punctuation=True,
generate_string_key="password",
secret_string_template=json.dumps(
{
'username': _config.Redshift.DATA_ENGINEER_USER,
'dbname': _config.Redshift.DATABASE,
'host': redshift_cluster_endpoint.hostname,
'port': core.Token.as_string(redshift_cluster_endpoint.port)
}
)
))
self.__quicksight_user_secret = Secret(
scope=self, id='DatavizUserSecret',
description="Quicksight user Redshift",
generate_secret_string=SecretStringGenerator(
exclude_characters="'",
exclude_punctuation=True,
generate_string_key="password",
secret_string_template=json.dumps(
{
'username': _config.Redshift.DATAVIZ_USER,
'dbname': _config.Redshift.DATABASE,
'host': redshift_cluster_endpoint.hostname,
'port': core.Token.as_string(redshift_cluster_endpoint.port)
}
)
))
self.__subnets_selection = _ec2.SubnetSelection(availability_zones=None, one_per_az=None,
subnet_group_name=None, subnet_name=None,
subnets=None, subnet_type=_ec2.SubnetType.PRIVATE)
# Use the following command line to generate the python dependencies layer content
# pip3 install -t lambda-layer/python/lib/python3.8/site-packages -r lambda/requirements.txt
# Build the lambda layer assets
subprocess.call(
['pip', 'install', '-t', 'dwh/dwh_cdk/bootstrap_lambda_layer/python/lib/python3.8/site-packages', '-r',
'dwh/dwh_cdk/bootstrap_lambda/requirements.txt', '--platform', 'manylinux1_x86_64', '--only-binary=:all:',
'--upgrade'])
requirements_layer = _lambda.LayerVersion(scope=self,
id='PythonRequirementsTemplate',
code=_lambda.Code.from_asset('dwh/dwh_cdk/bootstrap_lambda_layer'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_8])
# This lambda function will run SQL commands to setup Redshift users and tables
bootstrap_function_name = 'RedshiftBootstrap'
register_template_lambda = _lambda.Function(scope=self,
id='RegisterTemplate',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
'dwh/dwh_cdk/bootstrap_lambda'),
handler='redshift_setup.handler',
environment={
'SQL_SCRIPT_LOCATION': _config.BINARIES_LOCATION + self.SQL_SCRIPT_DIR,
'SECRET_ARN': self.__redshift_secret_arn,
'SQL_SCRIPT_FILES': _config.RedshiftDeploy.SQL_SCRIPT_FILES,
'ETL_SECRET': self.__etl_user_secret.secret_arn,
'DATAENG_SECRET': self.__dataengineer_user_secret.secret_arn,
'DATAVIZ_SECRET': self.__quicksight_user_secret.secret_arn,
'GLUE_DATABASE': self.__clean_glue_db.database_name,
'REDSHIFT_IAM_ROLE': self.__redshift_role_arn
},
layers=[requirements_layer],
timeout=core.Duration.minutes(3),
vpc=self.__vpc,
vpc_subnets=self.__subnets_selection,
security_group=self.__lambda_sg,
function_name=bootstrap_function_name,
memory_size=256
)
lambda_role = register_template_lambda.role
lambda_role.add_to_policy(PolicyStatement(
actions=['secretsmanager:GetResourcePolicy', 'secretsmanager:GetSecretValue',
'secretsmanager:DescribeSecret', 'secretsmanager:ListSecretVersionIds'],
resources=[stack.format_arn(service='secretsmanager', resource='*')]))
lambda_role.add_to_policy(PolicyStatement(actions=['logs:CreateLogGroup'],
resources=[stack.format_arn(service='logs', resource='*')]))
lambda_role.add_to_policy(PolicyStatement(actions=['logs:CreateLogStream', 'logs:PutLogEvents'],
resources=[stack.format_arn(service='logs', resource='log_group',
resource_name='/aws/lambda/' + bootstrap_function_name + ':*')]))
artifacts_bucket_arn = 'arn:aws:s3:::' + _config.ARA_BUCKET.replace("s3://", "")
lambda_role.add_to_policy(PolicyStatement(actions=['s3:GetObject', 's3:GetObjectVersion'],
resources=[artifacts_bucket_arn,
artifacts_bucket_arn + '/binaries/*']))
bootstrap_lambda_provider = Provider(scope=self,
id='BootstrapLambdaProvider',
on_event_handler=register_template_lambda)
CustomResource(scope=self,
id='ExecuteRegisterTemplate',
service_token=bootstrap_lambda_provider.service_token)
self.__secrets_manager_vpc_endpoint_sg = _ec2.SecurityGroup(self, id="secrets_manager_vpc_endpoint-sg",
vpc=self.__vpc, allow_all_outbound=None,
description=None,
security_group_name="secrets-manager-vpc_endpoint-sg")
self.__secrets_manager_vpc_endpoint_sg.add_ingress_rule(self.__lambda_sg,
_ec2.Port.all_tcp()
)
self.__security_groups_list = [self.__secrets_manager_vpc_endpoint_sg]
self.__endpoint_service_name = 'com.amazonaws.%s.secretsmanager' % stack.region
# Create VPC endpoint for SecretsManager
secrets_manager_vpc_endpoint = _ec2.InterfaceVpcEndpoint(stack, "Secretsmanager VPC Endpoint",
vpc=self.__vpc,
service=_ec2.InterfaceVpcEndpointService(
self.__endpoint_service_name, 443),
subnets=self.__subnets_selection,
security_groups=self.__security_groups_list
)
