def __init__(self, scope: core.Stack, id: str, **kwargs):
super().__init__(scope, id, **kwargs)
self.base_platform = BasePlatform(self, self.stack_name)
self.fargate_task_image = aws_ecs_patterns.ApplicationLoadBalancedTaskImageOptions(
image=aws_ecs.ContainerImage.from_registry(
"brentley/ecsdemo-frontend"),
container_port=3000,
environment={
"CRYSTAL_URL": "http://ecsdemo-crystal.service:3000/crystal",
"NODEJS_URL": "http://ecsdemo-nodejs.service:3000"
},
)
self.fargate_load_balanced_service = aws_ecs_patterns.ApplicationLoadBalancedFargateService(
self,
"FrontendFargateLBService",
cluster=self.base_platform.ecs_cluster,
cpu=256,
memory_limit_mib=512,
desired_count=1,
public_load_balancer=True,
cloud_map_options=self.base_platform.sd_namespace,
task_image_options=self.fargate_task_image)
self.fargate_load_balanced_service.service.connections.allow_to(
self.base_platform.services_sec_grp,
port_range=aws_ec2.Port(protocol=aws_ec2.Protocol.TCP,
string_representation="frontendtobackend",
from_port=3000,
to_port=3000))
def __init__(self, scope: core.Construct, id: str,
e2e_security_group: aws_ec2.SecurityGroup, vpc: aws_ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
cluster_sec_group = aws_ec2.SecurityGroup(self,
'msk-cluster-sec-group',
vpc=vpc)
cluster_sec_group.add_ingress_rule(peer=e2e_security_group,
connection=aws_ec2.Port(
string_representation='kafka',
protocol=aws_ec2.Protocol.TCP,
from_port=DEFAULT_KAFKA_PORT,
to_port=DEFAULT_KAFKA_PORT))
cluster = aws_msk.CfnCluster(
self,
'msk-cluster',
cluster_name='cdk-test',
number_of_broker_nodes=len(
vpc.private_subnets), # this is the minimum number needed
kafka_version='2.3.1',
broker_node_group_info=aws_msk.CfnCluster.
BrokerNodeGroupInfoProperty(
instance_type="kafka.m5.large",
client_subnets=[
subnet.subnet_id for subnet in vpc.private_subnets
],
security_groups=[cluster_sec_group.security_group_id]))
core.CfnOutput(self, "arn", value=cluster.ref)
def __init__(self, scope: core.Stack, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create VPC
self.vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_name='api-gateway/VPC')
# Create ECS Cluster
self.ecs_cluster = ecs.Cluster(self, "ECSCluster", vpc=self.vpc)
# This high level construct will build a docker image, ecr repo and connect the ecs service to allow pull access
self.container_image = ecr.DockerImageAsset(self,
"Image",
directory="./")
# Task definition details to define the frontend service container
self.task_def = ecs_patterns.NetworkLoadBalancedTaskImageOptions(
image=ecs.ContainerImage.from_ecr_repository(
repository=self.container_image.repository),
container_port=80,
enable_logging=True,
environment={"GIT_HASH": "12345"},
)
# Create the frontend service
self.python_service = ecs_patterns.NetworkLoadBalancedFargateService(
self,
"PythonService",
cpu=256,
memory_limit_mib=512,
cluster=self.ecs_cluster,
desired_count=1,
task_image_options=self.task_def,
public_load_balancer=False,
)
self.python_service.service.connections.allow_from_any_ipv4(
port_range=ec2.Port(
protocol=ec2.Protocol.ALL,
string_representation="All port 80",
from_port=80,
),
description="Allows traffic on port 80 from NLB")
# Create VPC Link from API Gateway to NLB
# TODO: Make api id dynamic
self.rest_api = apigw.RestApi.from_rest_api_id(
self, "APIGateway", rest_api_id="6znhu1vqp6")
# TODO: Create stage variable for vpc links
self.gateway_vpc_link = apigw.VpcLink(
self,
"VPCLink",
description="VPC Link from API Gateway to ECS Python Service",
targets=[self.python_service.load_balancer],
vpc_link_name="ECS_VPC_LINK")
def __init__(self, scope: core.Construct, id: str, *, app_env: str,
vpc: ec2.Vpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
security_group_params = {
'security_group_name': f'rds-{app_env}-test-security-group',
'description': f'security group for rds test({app_env})',
'vpc': vpc,
}
security_group = ec2.SecurityGroup(self, 'security-group',
**security_group_params)
security_group.add_ingress_rule(
peer=ec2.Peer.ipv4("54.125.156.2/32"),
connection=ec2.Port(
string_representation='random office',
protocol=ec2.Protocol.TCP,
from_port=5432,
to_port=5432,
))
password = Param.value_for_string_parameter(
self, f'/{app_env}/test/DATABASE_PASSWORD')
rds_params = {
'engine':
rds.DatabaseInstanceEngine.POSTGRES,
'database_name':
Param.value_for_string_parameter(self,
f'/{app_env}/test/DATABASE_NAME'),
'master_username':
Param.value_for_string_parameter(self,
f'/{app_env}/test/DATABASE_USER'),
'master_user_password':
core.SecretValue.plain_text(password),
'instance_class':
ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3,
ec2.InstanceSize.MICRO),
'instance_identifier':
f'{app_env}-test',
'backup_retention':
core.Duration.days(7),
'delete_automated_backups':
True,
'security_groups': [security_group],
'storage_type':
rds.StorageType.GP2,
'allocated_storage':
20,
'engine_version':
'11.5',
'vpc':
vpc,
}
self._rds = rds.DatabaseInstance(self, 'rds', **rds_params)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
nat_instance = aws_ec2.NatProvider.instance(
instance_type=aws_ec2.InstanceType('t3a.nano'))
vpc = aws_ec2.Vpc(self,
'VPC',
max_azs=1,
nat_gateway_provider=nat_instance)
cluster_security_group = aws_ec2.SecurityGroup(
self,
'ClusterSecurityGroup',
vpc=vpc,
allow_all_outbound=True,
description="Allow Glue Job to access redshift",
security_group_name="serverless-redshift-query-testing-redshift")
cluster = aws_redshift.Cluster(
self,
'Cluster',
cluster_name='serverless-redshift-query-testing',
master_user=aws_redshift.Login(master_username='******'),
vpc=vpc,
removal_policy=core.RemovalPolicy.DESTROY,
security_groups=[cluster_security_group])
app = aws_sam.CfnApplication(
self,
'RedshiftQueryGlueJob',
location=
'https://redshift-query.s3-eu-west-1.amazonaws.com/glue-job-template.yaml',
parameters={
'ClusterId': 'serverless-redshift-query-testing',
'SQLStatements': "select 1;",
'Loglevel': "ERROR"
})
glue_job_security_group_ref = app.get_att(
'Outputs.SecurityGroup').to_string()
glue_job_security_group = aws_ec2.SecurityGroup.from_security_group_id(
self, 'GlueSecurityGroup', glue_job_security_group_ref)
cluster_security_group.add_ingress_rule(
peer=glue_job_security_group,
connection=aws_ec2.Port(protocol=aws_ec2.Protocol.TCP,
from_port=5439,
to_port=5439,
string_representation='Redshift Port'))
app.node.add_dependency(cluster)
def provision(
self,
name: str,
eks_version: eks.KubernetesVersion,
private_api: bool,
vpc: ec2.Vpc,
bastion_sg: ec2.SecurityGroup,
):
eks_sg = ec2.SecurityGroup(
self.scope,
"EKSSG",
vpc=vpc,
security_group_name=f"{name}-EKSSG",
allow_all_outbound=False,
)
# Note: We can't tag the EKS cluster via CDK/CF: https://github.com/aws/aws-cdk/issues/4995
cluster = eks.Cluster(
self.scope,
"eks",
cluster_name=name,
vpc=vpc,
endpoint_access=eks.EndpointAccess.PRIVATE
if private_api else None,
vpc_subnets=[ec2.SubnetType.PRIVATE],
version=eks_version,
default_capacity=0,
security_group=eks_sg,
)
if bastion_sg:
cluster.cluster_security_group.add_ingress_rule(
peer=bastion_sg,
connection=ec2.Port(
protocol=ec2.Protocol("TCP"),
string_representation="API Access",
from_port=443,
to_port=443,
),
)
cdk.CfnOutput(self.scope,
"eks_cluster_name",
value=cluster.cluster_name)
cdk.CfnOutput(
self.scope,
"eks_kubeconfig_cmd",
value=
f"aws eks update-kubeconfig --name {cluster.cluster_name} --region {self.scope.region} --role-arn {cluster.kubectl_role.role_arn}",
)
return cluster
def add_security_group_rule(self,
sg_id: str,
protocol: _ec2.Protocol,
cidr_range: str = None,
prefix_list: str = None,
from_port: int = 0,
to_port: int = 0,
is_ingress: bool = True,
description: str = None):
"""add security group rule"""
if cidr_range is None:
cidr_range = self._vpc.vpc_cidr_block
if from_port != 0 and to_port == 0:
to_port = from_port
if prefix_list is not None:
peer = _ec2.Peer.prefix_list(prefix_list)
rule_id = f'{sg_id}_{protocol.name}_prefixlist_{from_port}_{to_port}'
else:
peer = _ec2.Peer.ipv4(cidr_range)
rule_id = f'{sg_id}_{protocol.name}_{cidr_range}_{from_port}_{to_port}'
if is_ingress:
self._security_groups[sg_id].add_ingress_rule(
peer=peer,
connection=_ec2.Port(string_representation=rule_id,
protocol=protocol,
from_port=from_port,
to_port=to_port),
description=description)
else:
self._security_groups[sg_id].add_egress_rule(
peer=peer,
connection=_ec2.Port(string_representation=rule_id,
protocol=protocol,
from_port=from_port,
to_port=to_port),
description=description)
def __configure_neptune(self) -> None:
self.subnet_group = n.CfnDBSubnetGroup(
self,
'SubnetGroup',
db_subnet_group_description='Portfolio Management',
db_subnet_group_name='portfoliomgmtsubnetgroup',
subnet_ids=[
net.subnet_id for net in self.vpc._select_subnet_objects(
subnet_group_name='PortfolioMgmt')
])
self.security_group = ec2.SecurityGroup(
self,
'SecGroup',
vpc=self.vpc,
allow_all_outbound=True,
description='Security group for PortfolioMgmt feature')
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='Neptune',
from_port=8182,
to_port=8182))
self.neptune_cluster = n.CfnDBCluster(
self,
'NeptuneCluster',
db_subnet_group_name=self.subnet_group.db_subnet_group_name,
deletion_protection=False,
iam_auth_enabled=False,
storage_encrypted=True,
db_cluster_identifier='portfoliomgmt',
vpc_security_group_ids=[self.security_group.security_group_id])
counter = 0
for net in self.vpc._select_subnet_objects(
subnet_group_name='PortfolioMgmt'):
az_name = net.availability_zone
counter += 1
self.neptune_instance = n.CfnDBInstance(
self,
'NeptuneInstance-' + str(counter),
availability_zone=az_name,
db_instance_identifier='portmgmt-instance-' + str(counter),
db_instance_class='db.t3.medium',
allow_major_version_upgrade=False,
auto_minor_version_upgrade=True,
db_cluster_identifier=self.neptune_cluster.
db_cluster_identifier,
db_subnet_group_name=self.subnet_group.db_subnet_group_name)
def addSg(self, sg, cidr, proto, string_representation, from_port,
to_port):
if (proto == "TCP"):
protocol = ec2.Protocol.TCP
elif (proto == "UDP"):
protocol = ec2.Protocol.UDP
elif (proto == "ICMP"):
protocol = ec2.Protocol.ICMP
sg.add_ingress_rule(peer=ec2.Peer.ipv4(cidr),
connection=ec2.Port(
protocol=protocol,
string_representation=string_representation,
from_port=from_port,
to_port=to_port))
def open_port(self, port: int, peer: IPeer, ingress: bool = True) -> None:
"""
Modifies a given security group by opening a specified port.
:param port: Port to open (allow traffic).
:param peer: Peer (a CIDR or another security group).
:param ingress: Specifies whether it is configured for ingress or egress traffic.
:return: No return.
"""
assert port is not None
assert peer is not None
assert ingress is not None
sg = self.__security_group
if ingress:
sg.add_ingress_rule(
peer=peer,
connection=aws_ec2.Port(
protocol=aws_ec2.Protocol.TCP,
string_representation=f'Ingress {port} rule.',
from_port=port,
to_port=port
)
)
else:
sg.add_egress_rule(
peer=peer,
connection=aws_ec2.Port(
protocol=aws_ec2.Protocol.TCP,
string_representation=f'Egress {port} rule.',
from_port=port,
to_port=port
)
)
def __init__(self, scope: core.Stack, id: str, **kwargs):
super().__init__(scope, id, **kwargs)
self.base_platform = BasePlatform(self, self.stack_name)
self.fargate_task_image = aws_ecs_patterns.ApplicationLoadBalancedTaskImageOptions(
image=aws_ecs.ContainerImage.from_registry(
"adam9098/ecsdemo-frontend"),
container_port=3000,
environment={
"CRYSTAL_URL": "http://ecsdemo-crystal.service:3000/crystal",
"NODEJS_URL": "http://ecsdemo-nodejs.service:3000",
"REGION": getenv('AWS_DEFAULT_REGION')
},
)
self.fargate_load_balanced_service = aws_ecs_patterns.ApplicationLoadBalancedFargateService(
self,
"FrontendFargateLBService",
service_name='ecsdemo-frontend',
cluster=self.base_platform.ecs_cluster,
cpu=256,
memory_limit_mib=512,
desired_count=3,
public_load_balancer=True,
cloud_map_options=self.base_platform.sd_namespace,
task_image_options=self.fargate_task_image)
self.fargate_load_balanced_service.task_definition.add_to_task_role_policy(
aws_iam.PolicyStatement(actions=['ec2:DescribeSubnets'],
resources=['*']))
self.fargate_load_balanced_service.service.connections.allow_to(
self.base_platform.services_sec_grp,
port_range=aws_ec2.Port(protocol=aws_ec2.Protocol.TCP,
string_representation="frontendtobackend",
from_port=3000,
to_port=3000))
# Enable Service Autoscaling
self.autoscale = self.fargate_load_balanced_service.service.auto_scale_task_count(
min_capacity=1, max_capacity=10)
self.autoscale.scale_on_cpu_utilization(
"CPUAutoscaling",
target_utilization_percent=50,
scale_in_cooldown=core.Duration.seconds(30),
scale_out_cooldown=core.Duration.seconds(30))
def __init__(self, scope: core.Construct, id: str, vpc: ec2.IVpc, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.vpc = vpc
self.security_group = ec2.SecurityGroup(
self, 'EndpointSecurity',
vpc=vpc,
allow_all_outbound=True,
description='SG for AWS Resources in isolated subnet')
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(
protocol=ec2.Protocol.ALL,
string_representation='Any source'))
def __init__(self, scope: core.Construct, id: str, nw_stack: core.Stack,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
ec2_sg = _ec2.SecurityGroup(self,
id='test-ec2-instance-sg',
vpc=nw_stack.app_vpc)
bastion_sg = _ec2.SecurityGroup(self,
id='bastion-sg',
vpc=nw_stack.app_vpc)
prv = _ec2.Instance(self,
id='tgw_poc_instance',
instance_type=_ec2.InstanceType('t3a.nano'),
machine_image=_ec2.AmazonLinuxImage(),
key_name=EC2Stack.KEY_PAIR,
security_group=ec2_sg,
instance_name='tgw_nat_test_instance',
vpc=nw_stack.app_vpc,
vpc_subnets=_ec2.SubnetSelection(
subnet_type=_ec2.SubnetType.ISOLATED))
bastion = _ec2.Instance(self,
id='tgw_poc_bastion',
instance_type=_ec2.InstanceType('t3a.nano'),
machine_image=_ec2.AmazonLinuxImage(),
key_name=EC2Stack.KEY_PAIR,
security_group=bastion_sg,
instance_name='tgw_test_bastion',
vpc=nw_stack.app_vpc,
vpc_subnets=_ec2.SubnetSelection(
subnet_type=_ec2.SubnetType.PUBLIC))
ssh_port = _ec2.Port(protocol=_ec2.Protocol.TCP,
string_representation="tcp_22",
from_port=EC2Stack.SSH_PORT,
to_port=EC2Stack.SSH_PORT)
bastion_sg.add_ingress_rule(peer=_ec2.Peer.ipv4(EC2Stack.SSH_IP),
connection=ssh_port,
description='Allow SSH access from SSH_IP')
ec2_sg.add_ingress_rule(
peer=bastion_sg,
connection=ssh_port,
description='Allow SSH access from bastion host')
def __init__(self, scope: core.Construct, id: str, bmt_vpc: ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
db_subnet_group = rds.SubnetGroup(
self,
'Aurora',
description='aurora subnet group',
vpc=bmt_vpc,
removal_policy=core.RemovalPolicy.DESTROY,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED))
db_security_group = ec2.SecurityGroup(self, 'aurora-sg', vpc=bmt_vpc)
db_security_group.add_ingress_rule(
peer=ec2.Peer.ipv4('10.100.0.0/16'),
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="to allow from the vpc internal",
from_port=3306,
to_port=3306))
param_group = rds.ParameterGroup(
self,
'bmt-aurora-param',
engine=rds.DatabaseClusterEngine.AURORA_MYSQL)
param_group.add_parameter("performance_schema", "1")
rds.DatabaseCluster(
self,
'bmt-aurora-cluster',
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_2_07_1),
instance_props=rds.InstanceProps(
vpc=bmt_vpc,
instance_type=ec2.InstanceType.of(
instance_class=ec2.InstanceClass.BURSTABLE3,
instance_size=ec2.InstanceSize.MEDIUM),
security_groups=[db_security_group]),
instances=1,
subnet_group=db_subnet_group,
parameter_group=param_group,
removal_policy=core.RemovalPolicy.DESTROY)
def __init__(self, scope: core.Construct, id: str, vpc: aws_ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
_subnets = []
_subnets.append(
aws_ec2.Subnet(self,
'sbn-sourcedb-1',
availability_zone=vpc.availability_zones[0],
vpc_id=vpc.vpc_id,
cidr_block='10.0.1.0/24'))
_subnets.append(
aws_ec2.Subnet(self,
'sbn-sourcedb-2',
availability_zone=vpc.availability_zones[1],
vpc_id=vpc.vpc_id,
cidr_block='10.0.2.0/24'))
db_security_group = aws_ec2.SecurityGroup(self, 'sg-src-db', vpc=vpc)
db_security_group.add_ingress_rule(
peer=aws_ec2.Peer.ipv4('10.0.5.192/26'),
connection=aws_ec2.Port(
protocol=aws_ec2.Protocol.TCP,
string_representation="to allow traffic from the cmd",
from_port=5432,
to_port=5432))
_postgres_instance = rds.DatabaseInstance(
self,
'src_ora',
engine=rds.DatabaseInstanceEngine.POSTGRES,
vpc=vpc,
vpc_subnets=aws_ec2.SubnetSelection(subnets=_subnets),
security_groups=[db_security_group])
core.CfnOutput(self,
'secret_name',
value=_postgres_instance.secret.secret_name)
self._secret_name = _postgres_instance.secret.secret_name
self._secret_arn = _postgres_instance.secret.secret_arn
def __init__(self, scope: core.Construct, id: str, vpc: ec2.IVpc,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.security_group = ec2.SecurityGroup(
self,
'EndpointSecurity',
vpc=vpc,
allow_all_outbound=True,
description='SG for AWS Resources in isolated subnet')
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.ALL,
string_representation='Any source'))
self.gateways = {}
for svc in ['s3', 'dynamodb']:
self.gateways[svc] = ec2.GatewayVpcEndpoint(
self,
svc,
vpc=vpc,
service=ec2.GatewayVpcEndpointAwsService(name=svc))
self.interfaces = {}
for svc in [
'ssm', 'ec2messages', 'ec2', 'ssmmessages', 'kms',
'elasticloadbalancing', 'elasticfilesystem', 'lambda',
'states', 'events', 'execute-api', 'kinesis-streams',
'kinesis-firehose', 'logs', 'sns', 'sqs', 'secretsmanager',
'config', 'ecr.api', 'ecr.dkr'
]:
self.interfaces[svc] = ec2.InterfaceVpcEndpoint(
self,
svc,
vpc=vpc,
service=ec2.InterfaceVpcEndpointAwsService(name=svc),
open=True,
private_dns_enabled=True,
lookup_supported_azs=False,
security_groups=[self.security_group])
def get_security_group(self, ec2_params):
security_group = None
if 'security_group' in ec2_params:
security_group = ec2_params['security_group']
else:
if 'security_group_id' in ec2_params and ec2_params[
'security_group_id']:
security_group = ec2.SecurityGroup.from_security_group_id(
self,
"SecurityGroup",
security_group_id=ec2_params['security_group_id'],
mutable=False)
else:
security_group = ec2.SecurityGroup(self,
"SecurityGroup",
vpc=ec2_params['vpc'])
security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(string_representation="sr",
protocol=ec2.Protocol("UDP"),
from_port=ec2_params['from_port'],
to_port=ec2_params['to_port']))
return security_group
def __init__(self, scope: core.Construct, id: str, config_dict,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
""" Get VPC details """
vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id=config_dict['vpc_id'])
""" Create Security Group for Batch Env """
batch_security_group = "datalake-batch-security-group"
createBatchSecurityGroup = ec2.SecurityGroup(
self,
"createBatchSecurityGroup",
vpc=vpc,
allow_all_outbound=True,
description=
"This security group will be used for AWS Batch Compute Env",
security_group_name=batch_security_group)
createBatchSecurityGroup.add_ingress_rule(
peer=ec2.Peer.ipv4("0.0.0.0/0"),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="ingress_rule",
from_port=22,
to_port=22))
createBatchSecurityGroup.add_egress_rule(
peer=ec2.Peer.ipv4("0.0.0.0/0"),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="egress_rule",
from_port=-1,
to_port=-1))
core.CfnOutput(self,
"createBatchSecurityGroupId",
value=createBatchSecurityGroup.security_group_id)
""" Create IAM Role for ecsInstance """
createECSInstanceRole = iam.Role(
self,
"createECSInstanceRole",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
description=
"This instance role will be used by the ECS cluster instances",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2FullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AWSBatchFullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"SecretsManagerReadWrite"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonAthenaFullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/"
"AmazonEC2ContainerServiceforEC2Role"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSBatchServiceRole")
],
role_name="datalake-ecsinstance-role")
createInstanceProfile = iam.CfnInstanceProfile(
self,
"createInstanceProfile",
roles=[createECSInstanceRole.role_name],
instance_profile_name="datalake-ecsinstance-role")
useECSInstanceProfile = createInstanceProfile.instance_profile_name
core.CfnOutput(self,
"createECSInstanceRoleName",
value=createECSInstanceRole.role_name)
""" Create Spot Fleet Role """
createSpotFleetRole = iam.Role(
self,
'createSpotFleetRole',
assumed_by=iam.ServicePrincipal("spotfleet.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2SpotFleetTaggingRole")
])
core.CfnOutput(self,
"createSpotFleetRoleName",
value=createSpotFleetRole.role_name)
useSpotFleetRole = createSpotFleetRole.without_policy_updates()
""" Create Batch Service Role """
createBatchServiceRole = iam.Role(
self,
'createBatchServiceRole',
assumed_by=iam.ServicePrincipal("batch.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSBatchServiceRole")
])
core.CfnOutput(self,
"createBatchServiceRoleName",
value=createBatchServiceRole.role_name)
useBatchServiceRole = createBatchServiceRole.without_policy_updates()
""" Create Compute Environment """
subnet_1 = ec2.Subnet.from_subnet_attributes(
self,
"subnet_1",
subnet_id=config_dict['SubnetIds'].split(",")[0],
availability_zone=config_dict['AvailabilityZones'].split(",")[0])
subnet_2 = ec2.Subnet.from_subnet_attributes(
self,
"subnet_2",
subnet_id=config_dict['SubnetIds'].split(",")[1],
availability_zone=config_dict['AvailabilityZones'].split(",")[1])
createBatchComputeEnv = batch.ComputeEnvironment(
self,
"createBatchComputeEnv",
compute_environment_name="datalake-compute-env",
service_role=useBatchServiceRole,
compute_resources=batch.ComputeResources(
vpc=vpc,
type=batch.ComputeResourceType.SPOT,
bid_percentage=60,
desiredv_cpus=0,
maxv_cpus=100,
minv_cpus=0,
security_groups=[createBatchSecurityGroup],
vpc_subnets=ec2.SubnetSelection(subnets=[subnet_1, subnet_2]),
instance_role=useECSInstanceProfile,
spot_fleet_role=useSpotFleetRole,
compute_resources_tags=core.Tag.add(
self, 'Name', 'Datalake Pipeline Instance')))
core.CfnOutput(self,
"createBatchComputeEnvName",
value=createBatchComputeEnv.compute_environment_name)
getIComputeEnvObject = batch.ComputeEnvironment.from_compute_environment_arn(
self,
"getComputeEnvAtrributes",
compute_environment_arn=createBatchComputeEnv.
compute_environment_arn)
""" Create Batch Job Queue """
createBatchJobQueue = batch.JobQueue(
self,
"createBatchJobQueue",
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=getIComputeEnvObject, order=1)
],
enabled=True,
job_queue_name="datalake-job-queue",
priority=1)
core.CfnOutput(self,
"createBatchJobQueueName",
value=createBatchJobQueue.job_queue_name)
""" Create ECR Repo for datalake images """
createECRRepo = ecr.Repository(
self,
"createECRRepo",
repository_name=config_dict['workflow_ecr_repo'])
core.CfnOutput(self,
"createECRRepoName",
value=createECRRepo.repository_name)
def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# -- VPC
vpc = ec2.Vpc(self, "vpc_airflow")
# ecr
ecr_repo = ecr.Repository.from_repository_name(self,
"ecr_repo_airflow",
"airflow")
# rds
sg_airflow_backend_db = ec2.SecurityGroup(
self,
"sg_airflow_backend_database",
vpc=vpc,
description="Airflow backend database",
security_group_name="sg_airflow_backend_database",
)
db = rds.DatabaseInstance(
self,
"rds_airfow_backend",
master_username="******",
master_user_password=core.SecretValue.plain_text("postgres"),
database_name="airflow",
engine=rds.DatabaseInstanceEngine.postgres(
version=rds.PostgresEngineVersion.VER_11_8),
vpc=vpc,
instance_type=ec2.InstanceType.of(
ec2.InstanceClass.BURSTABLE3,
ec2.InstanceSize.MICRO,
),
instance_identifier="airflow-backend",
removal_policy=core.RemovalPolicy.DESTROY,
deletion_protection=False,
security_groups=[sg_airflow_backend_db],
vpc_placement=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC),
)
# -- ElasticCache Redis
sg_redis = ec2.SecurityGroup(
self,
"sg_redis",
vpc=vpc,
description="Airflow redis",
security_group_name="sg_redis",
)
redis_subnet_group = ec.CfnSubnetGroup(
self,
"airflow-redis-subnet-group",
description="For Airflow Task Queue",
subnet_ids=vpc.select_subnets(
subnet_type=ec2.SubnetType.PRIVATE).subnet_ids,
cache_subnet_group_name="airflow-redis-task-queue",
)
redis = ec.CfnCacheCluster(
self,
"redis",
cluster_name="airflow-redis",
cache_node_type="cache.t2.micro",
engine="redis",
num_cache_nodes=1,
auto_minor_version_upgrade=True,
engine_version="5.0.6",
port=REDIS_PORT,
cache_subnet_group_name=redis_subnet_group.ref,
vpc_security_group_ids=[sg_redis.security_group_id],
)
# ECS cluster
cluster = ecs.Cluster(
self,
"ecs_airflow",
cluster_name="airflow",
vpc=vpc,
container_insights=True,
)
# scheduler
scheduler_task_role = iam.Role(
self,
"iam_role_scheduler",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
description="IAM role for ECS Scheduler service",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2ContainerRegistryReadOnly"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchLogsFullAccess"),
],
role_name="airflow-ecs-scheduler-task",
)
scheduler_task = ecs.FargateTaskDefinition(
self,
"ecs_task_scheduler",
cpu=512,
memory_limit_mib=2048,
task_role=scheduler_task_role,
)
scheduler_task.add_container(
"scheduler",
command=["scheduler"],
# credentials should be provided from Secrets Manager
environment={
"LOAD_EX": "n",
"FERNET_KEY": "46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=",
"EXECUTOR": "Celery",
"POSTGRES_HOST": db.db_instance_endpoint_address,
"POSTGRES_USER": "******",
"POSTGRES_PASSWORD": "******",
"POSTGRES_DB": "airflow",
"REDIS_HOST": redis.attr_redis_endpoint_address,
},
image=ecs.ContainerImage.from_ecr_repository(
ecr_repo,
"1.10.9",
),
logging=ecs.LogDriver.aws_logs(
stream_prefix="scheduler",
log_group=logs.LogGroup(
self,
"log-airflow-scheduler",
log_group_name="ecs/airflow/scheduler",
retention=logs.RetentionDays.ONE_WEEK,
),
),
)
sg_airflow_scheduler = ec2.SecurityGroup(
self,
"sg_airflow_scheduler",
vpc=vpc,
description="Airflow Scheduler service",
security_group_name="sg_airflow_scheduler",
)
sg_redis.add_ingress_rule(
peer=sg_airflow_scheduler,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from scheduler",
from_port=REDIS_PORT,
to_port=REDIS_PORT,
),
description="from scheduler service",
)
sg_airflow_backend_db.add_ingress_rule(
peer=sg_airflow_scheduler,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from home",
from_port=POSTGRES_PORT,
to_port=POSTGRES_PORT,
),
description="home",
)
scheduler_service = ecs.FargateService(
self,
"ecs_service_scheduler",
cluster=cluster,
task_definition=scheduler_task,
desired_count=1,
security_groups=[sg_airflow_scheduler],
service_name="scheduler",
)
# flower
flower_task_role = iam.Role(
self,
"iam_role_flower",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
description="IAM role for ECS Flower service",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2ContainerRegistryReadOnly"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchLogsFullAccess"),
],
role_name="airflow-ecs-flower-task",
)
flower_task = ecs.FargateTaskDefinition(
self,
"ecs_task_flower",
cpu=512,
memory_limit_mib=1024,
task_role=scheduler_task_role,
)
flower_task.add_container(
"flower",
command=["flower"],
# credentials should be provided from Secrets Manager
environment={
"LOAD_EX": "n",
"FERNET_KEY": "46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=",
"EXECUTOR": "Celery",
"REDIS_HOST": redis.attr_redis_endpoint_address,
},
image=ecs.ContainerImage.from_ecr_repository(
ecr_repo,
"1.10.9",
),
logging=ecs.LogDriver.aws_logs(
stream_prefix="flower",
log_group=logs.LogGroup(
self,
"log-airflow-flower",
log_group_name="ecs/airflow/flower",
retention=logs.RetentionDays.ONE_WEEK,
),
),
).add_port_mappings(
ecs.PortMapping(
container_port=FLOWER_PORT,
host_port=FLOWER_PORT,
protocol=ecs.Protocol.TCP,
))
sg_airflow_flower = ec2.SecurityGroup(
self,
"sg_airflow_flower",
vpc=vpc,
description="Airflow Flower service",
security_group_name="sg_airflow_flower",
)
sg_airflow_flower.add_ingress_rule(
peer=ec2.Peer.ipv4("115.66.217.45/32"),
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from homr",
from_port=FLOWER_PORT,
to_port=FLOWER_PORT,
),
description="from home",
)
sg_redis.add_ingress_rule(
peer=sg_airflow_flower,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from flower",
from_port=REDIS_PORT,
to_port=REDIS_PORT,
),
description="from flower",
)
flower_service = ecs.FargateService(
self,
"ecs_service_flower",
cluster=cluster,
task_definition=flower_task,
desired_count=1,
security_groups=[sg_airflow_flower],
service_name="flower",
)
# worker
worker_task_role = iam.Role(
self,
"iam_role_worker",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
description="IAM role for ECS worker service",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2ContainerRegistryReadOnly"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchLogsFullAccess"),
],
role_name="airflow-ecs-worker-task",
)
worker_task = ecs.FargateTaskDefinition(
self,
"ecs_task_worker",
cpu=1024,
memory_limit_mib=3072,
task_role=worker_task_role,
)
worker_task.add_container(
"worker",
command=["worker"],
# credentials should be provided from Secrets Manager
environment={
"LOAD_EX": "n",
"FERNET_KEY": "46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=",
"EXECUTOR": "Celery",
"POSTGRES_HOST": db.db_instance_endpoint_address,
"POSTGRES_USER": "******",
"POSTGRES_PASSWORD": "******",
"POSTGRES_DB": "airflow",
"REDIS_HOST": redis.attr_redis_endpoint_address,
},
image=ecs.ContainerImage.from_ecr_repository(
ecr_repo,
"1.10.9",
),
logging=ecs.LogDriver.aws_logs(
stream_prefix="worker",
log_group=logs.LogGroup(
self,
"log-airflow-worker",
log_group_name="ecs/airflow/worker",
retention=logs.RetentionDays.ONE_WEEK,
),
),
)
sg_airflow_worker = ec2.SecurityGroup(
self,
"sg_airflow_worker",
vpc=vpc,
description="Airflow worker service",
security_group_name="sg_airflow_worker",
)
sg_redis.add_ingress_rule(
peer=sg_airflow_worker,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from worker",
from_port=REDIS_PORT,
to_port=REDIS_PORT,
),
description="from worker service",
)
sg_airflow_backend_db.add_ingress_rule(
peer=sg_airflow_worker,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from worker",
from_port=POSTGRES_PORT,
to_port=POSTGRES_PORT,
),
description="From worker",
)
worker_service = ecs.FargateService(
self,
"ecs_service_worker",
cluster=cluster,
task_definition=worker_task,
desired_count=1,
security_groups=[sg_airflow_worker],
service_name="worker",
)
# web server
web_server_task_role = iam.Role(
self,
"iam_role_web_server",
assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"),
description="IAM role for ECS web server service",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2ContainerRegistryReadOnly"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchLogsFullAccess"),
],
role_name="airflow-ecs-web-server-task",
)
web_server_task = ecs.FargateTaskDefinition(
self,
"ecs_task_web_server",
cpu=512,
memory_limit_mib=1024,
task_role=web_server_task_role,
)
web_server_task.add_container(
"web_server",
command=["webserver"],
# credentials should be provided from Secrets Manager
environment={
"LOAD_EX": "n",
"FERNET_KEY": "46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=",
"EXECUTOR": "Celery",
"POSTGRES_HOST": db.db_instance_endpoint_address,
"POSTGRES_USER": "******",
"POSTGRES_PASSWORD": "******",
"POSTGRES_DB": "airflow",
"REDIS_HOST": redis.attr_redis_endpoint_address,
},
image=ecs.ContainerImage.from_ecr_repository(
ecr_repo,
"1.10.9",
),
logging=ecs.LogDriver.aws_logs(
stream_prefix="web_server",
log_group=logs.LogGroup(
self,
"log-airflow-web-server",
log_group_name="ecs/airflow/web-server",
retention=logs.RetentionDays.ONE_WEEK,
),
),
).add_port_mappings(
ecs.PortMapping(
container_port=WEB_SERVER_PORT,
host_port=WEB_SERVER_PORT,
protocol=ecs.Protocol.TCP,
))
sg_airflow_web_server = ec2.SecurityGroup(
self,
"sg_airflow_web_server",
vpc=vpc,
description="Airflow web server service",
security_group_name="sg_airflow_web_server",
)
sg_airflow_backend_db.add_ingress_rule(
peer=sg_airflow_web_server,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="From web server",
from_port=POSTGRES_PORT,
to_port=POSTGRES_PORT,
),
description="From web server",
)
sg_airflow_backend_db.add_ingress_rule(
peer=sg_airflow_web_server,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="From web server",
from_port=POSTGRES_PORT,
to_port=POSTGRES_PORT,
),
description="From web server",
)
sg_redis.add_ingress_rule(
peer=sg_airflow_web_server,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="from web server",
from_port=REDIS_PORT,
to_port=REDIS_PORT,
),
description="from web server",
)
web_server_service = ecs.FargateService(
self,
"ecs_service_web_server",
cluster=cluster,
task_definition=web_server_task,
desired_count=1,
security_groups=[sg_airflow_web_server],
service_name="web_server",
)
# Load balancer
sg_airflow_alb = ec2.SecurityGroup(
self,
"sg_airflow_alb",
vpc=vpc,
description="Airflow ALB",
security_group_name="sg_airflow_alb",
)
# ALB -> web server
sg_airflow_web_server.add_ingress_rule(
peer=sg_airflow_alb,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="From ALB",
from_port=WEB_SERVER_PORT,
to_port=WEB_SERVER_PORT,
),
description="From ALB",
)
# ALB -> flower
sg_airflow_flower.add_ingress_rule(
peer=sg_airflow_alb,
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="From ALB",
from_port=FLOWER_PORT,
to_port=FLOWER_PORT,
),
description="From ALB",
)
# Home -> ALB
sg_airflow_alb.add_ingress_rule(
peer=ec2.Peer.ipv4(MY_IP_CIDR),
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="From Home",
from_port=ALB_PORT,
to_port=ALB_PORT,
),
description="From Home",
)
# Home -> ALB
sg_airflow_alb.add_ingress_rule(
peer=ec2.Peer.ipv4(MY_IP_CIDR),
connection=ec2.Port(
protocol=ec2.Protocol.TCP,
string_representation="From Home",
from_port=FLOWER_PORT,
to_port=FLOWER_PORT,
),
description="From Home",
)
alb = elb.ApplicationLoadBalancer(
self,
"alb_airflow",
internet_facing=True,
security_group=sg_airflow_alb,
vpc=vpc,
load_balancer_name="alb-airflow",
)
listener1 = alb.add_listener(
"alb_airflow_listener1",
open=False,
port=ALB_PORT,
protocol=elb.ApplicationProtocol.HTTP,
default_target_groups=[
elb.ApplicationTargetGroup(
self,
"alb_airflow_target_group_web_server",
port=WEB_SERVER_PORT,
protocol=elb.ApplicationProtocol.HTTP,
target_group_name="alb-tg-airflow-web-server",
targets=[web_server_service],
vpc=vpc,
)
],
)
alb.add_listener(
"alb_airflow_listener2",
open=False,
port=FLOWER_PORT,
protocol=elb.ApplicationProtocol.HTTP,
default_target_groups=[
elb.ApplicationTargetGroup(
self,
"alb_airflow_target_group_flower",
port=FLOWER_PORT,
protocol=elb.ApplicationProtocol.HTTP,
target_group_name="alb-tg-aiflow-flower",
targets=[flower_service],
vpc=vpc,
)
],
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# EC2 Vpc construct
vpc = ec2.Vpc(
self,
id="cromwell_server_vpc",
max_azs=2
)
# ECS Cluster construct
cluster = ecs.Cluster(
self,
id="cromwell_cluster",
vpc=vpc
)
# IAM roles
ecstaskexecutionrole = iam.Role.from_role_arn(
self,
"ecstaskexecutionrole",
role_arn="arn:aws:iam::562965587442:role/ecsTaskExecutionRole"
)
batch_service_role = iam.Role.from_role_arn(
self,
"batchservicerole",
role_arn="arn:aws:iam::562965587442:role/AWSBatchServiceRole"
)
fargate_cromwell_role = iam.Role.from_role_arn(
self,
"fargate_cromwell_role",
role_arn="arn:aws:iam::562965587442:role/fargate_cromwell_role"
)
# Cromwell docker image from ECR
container_img = ecr.Repository.from_repository_name(
self,
"cromwell_docker_image",
repository_name=CROMWELL_REPOSITORY_NAME
)
# ECS task definition construct
task_def = ecs.TaskDefinition(
self,
"cromwell_server_task",
execution_role=ecstaskexecutionrole,
task_role=fargate_cromwell_role,
compatibility=ecs.Compatibility.FARGATE,
cpu="1024",
memory_mib="4096"
)
# ECS container definition construct
container_def = ecs.ContainerDefinition(
self,
"cromwell_container",
task_definition=task_def,
image=ecs.ContainerImage.from_ecr_repository(
repository=container_img,
tag=CROMWELL_IMAGE_TAG
),
command=["bash", "run_cromwell_server.sh"],
cpu=1,
health_check=None,
working_directory='/',
logging=ecs.LogDriver.aws_logs(
stream_prefix="cromwell_logs",
datetime_format=None,
log_group=None,
log_retention=None,
multiline_pattern=None
)
)
container_def.add_port_mappings(
ecs.PortMapping(
container_port=CROMWELL_PORT_NUMBER,
host_port=CROMWELL_PORT_NUMBER,
protocol=ecs.Protocol.TCP
)
)
# EC2 Security Group construct
security_group = ec2.SecurityGroup(
self,
"cromwell_server_security_group",
vpc=vpc,
allow_all_outbound=True,
security_group_name="cromwell_server_security_group",
description="This is the security group assigned to the cromwell server running as a Fargate service.",
)
security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ecs.Protocol.TCP,
from_port=CROMWELL_PORT_NUMBER,
to_port=CROMWELL_PORT_NUMBER,
string_representation="cromwell_server_port"),
remote_rule=None
)
# ECS Fargate Service construct
service = ecs.FargateService(
self,
"cromwell_service",
task_definition=task_def,
cluster=cluster,
service_name="cromwell_server_service",
assign_public_ip=True,
desired_count=1,
security_group=security_group
)
# Batch resources
# Reference:
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-batch-computeenvironment-computeresources.html
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-launchtemplatedata.html
# with open("lib/aws_batch_launch_template_user_data.txt", 'r') as user_data_file:
# user_data = user_data_file.read()
#
# ec2_user_data = ec2.UserData.custom(content=user_data)
# ec2_instance =ec2.Instance(
# self,
# "ec2Instance",
# instance_type=ec2.InstanceType("t2.small"),
# machine_image=ec2.AmazonLinuxImage(),
# vpc=vpc,
# user_data=ec2_user_data
# )
# launch_template_data = core.CfnResource(
# self,
# "cromwell_launch_template_data",
# type="AWS::EC2::LaunchTemplate.LaunchTemplateData",
# properties={
# "UserData": user_data
# }
# )
#
# launch_template = ec2.CfnLaunchTemplate(
# self,
# "cromwell_launch_template",
# launch_template_name="cromwell_launch_template",
# launch_template_data=launch_template_data,
# )
#
# compute_resources = core.CfnResource(
# self,
# "cromwell_compute_resources",
# type="AWS::Batch::ComputeEnvironment.ComputeResources",
# properties={
# "DesiredvCpus": 256,
# "Ec2KeyPair": "genovic-qc-eddev",
# "InstanceRole": "arn:aws:iam::562965587442:role/ecsInstanceRole",
# "InstanceTypes": ["optimal"],
# "LaunchTemplate": launch_template.launch_template_name,
# "MaxvCpus": 256,
# "MinvCpus": 0,
# "SecurityGroupIds": [vpc.vpc_default_security_group],
# "Subnets": [subnet.subnet_id for subnet in vpc.public_subnets],
# "Tags": "cromwell_compute_resource",
# "Type": "EC2"
# }
# )
#
# compute_env = batch.CfnComputeEnvironment(
# self,
# "cromwell_compute_env",
# service_role=batch_service_role,
# compute_environment_name="cromwell_compute_env",
# type="MANAGED",
# state="ENABLED",
# compute_resources=compute_resources
# )
#
# queue = batch.CfnJobQueue(
# self,
# "cromwell_queue",
# compute_environment_order=compute_env,
# priority=1,
# job_queue_name="cromwell_queue",
# state="ENABLED"
# )
#
# core.CfnOutput(
# self,
# "cromwell_queue_name",
# value=queue.job_queue_name
# )
core.CfnOutput(
self,
"FargateCromwellServiceArn",
value=service.service_arn
)
def appmesh(self):
# This will create the app mesh (control plane)
self.mesh = aws_appmesh.Mesh(self,
"EcsWorkShop-AppMesh",
mesh_name="ecs-mesh")
# We will create a App Mesh Virtual Gateway
self.mesh_vgw = aws_appmesh.VirtualGateway(
self,
"Mesh-VGW",
mesh=self.mesh,
listeners=[aws_appmesh.VirtualGatewayListener.http(port=3000)],
virtual_gateway_name="ecsworkshop-vgw")
# Creating the mesh gateway task for the frontend app
# For more info related to App Mesh Proxy check https://docs.aws.amazon.com/app-mesh/latest/userguide/getting-started-ecs.html
self.mesh_gw_proxy_task_def = aws_ecs.FargateTaskDefinition(
self,
"mesh-gw-proxy-taskdef",
cpu=256,
memory_limit_mib=512,
family="mesh-gw-proxy-taskdef",
)
# LogGroup for the App Mesh Proxy Task
self.logGroup = aws_logs.LogGroup(
self,
"ecsworkshopMeshGateway",
#log_group_name="ecsworkshop-mesh-gateway",
retention=aws_logs.RetentionDays.ONE_WEEK)
# App Mesh Virtual Gateway Envoy proxy Task definition
# For a use specific ECR region, please check https://docs.aws.amazon.com/app-mesh/latest/userguide/envoy.html
container = self.mesh_gw_proxy_task_def.add_container(
"mesh-gw-proxy-contdef",
image=aws_ecs.ContainerImage.from_registry(
"public.ecr.aws/appmesh/aws-appmesh-envoy:v1.18.3.0-prod"),
container_name="envoy",
memory_reservation_mib=256,
environment={
"REGION": getenv('AWS_DEFAULT_REGION'),
"ENVOY_LOG_LEVEL": "info",
"ENABLE_ENVOY_STATS_TAGS": "1",
# "ENABLE_ENVOY_XRAY_TRACING": "1",
"APPMESH_RESOURCE_ARN": self.mesh_vgw.virtual_gateway_arn
},
essential=True,
logging=aws_ecs.LogDriver.aws_logs(stream_prefix='/mesh-gateway',
log_group=self.logGroup),
health_check=aws_ecs.HealthCheck(command=[
"CMD-SHELL",
"curl -s http://localhost:9901/server_info | grep state | grep -q LIVE"
], ))
# Default port where frontend app is listening
container.add_port_mappings(aws_ecs.PortMapping(container_port=3000))
#ammmesh-xray-uncomment
# xray_container = self.mesh_gw_proxy_task_def.add_container(
# "FrontendServiceXrayContdef",
# image=aws_ecs.ContainerImage.from_registry("amazon/aws-xray-daemon"),
# logging=aws_ecs.LogDriver.aws_logs(
# stream_prefix='/xray-container',
# log_group=self.logGroup
# ),
# essential=True,
# container_name="xray",
# memory_reservation_mib=256,
# user="******"
# )
# container.add_container_dependencies(aws_ecs.ContainerDependency(
# container=xray_container,
# condition=aws_ecs.ContainerDependencyCondition.START
# )
# )
#ammmesh-xray-uncomment
# For environment variables check https://docs.aws.amazon.com/app-mesh/latest/userguide/envoy-config.html
self.mesh_gateway_proxy_fargate_service = aws_ecs_patterns.NetworkLoadBalancedFargateService(
self,
"MeshGW-Proxy-Fargate-Service",
service_name='mesh-gw-proxy',
cpu=256,
memory_limit_mib=512,
desired_count=1,
listener_port=80,
assign_public_ip=True,
task_definition=self.mesh_gw_proxy_task_def,
cluster=self.ecs_cluster,
public_load_balancer=True,
cloud_map_options=aws_ecs.CloudMapOptions(
cloud_map_namespace=self.ecs_cluster.
default_cloud_map_namespace,
name='mesh-gw-proxy'))
# For testing purposes we will open any ipv4 requests to port 3000
self.mesh_gateway_proxy_fargate_service.service.connections.allow_from_any_ipv4(
port_range=aws_ec2.Port(protocol=aws_ec2.Protocol.TCP,
string_representation="vtw_proxy",
from_port=3000,
to_port=3000),
description="Allow NLB connections on port 3000")
self.mesh_gw_proxy_task_def.default_container.add_ulimits(
aws_ecs.Ulimit(hard_limit=15000,
name=aws_ecs.UlimitName.NOFILE,
soft_limit=15000))
#Adding necessary policies for Envoy proxy to communicate with required services
self.mesh_gw_proxy_task_def.execution_role.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2ContainerRegistryReadOnly"))
self.mesh_gw_proxy_task_def.execution_role.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchLogsFullAccess"))
self.mesh_gw_proxy_task_def.task_role.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchFullAccess"))
# self.mesh_gw_proxy_task_def.task_role.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("AWSXRayDaemonWriteAccess"))
self.mesh_gw_proxy_task_def.task_role.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AWSAppMeshEnvoyAccess"))
self.mesh_gw_proxy_task_def.execution_role.add_to_policy(
aws_iam.PolicyStatement(actions=['ec2:DescribeSubnets'],
resources=['*']))
core.CfnOutput(self,
"MeshGwNlbDns",
value=self.mesh_gateway_proxy_fargate_service.
load_balancer.load_balancer_dns_name,
export_name="MeshGwNlbDns")
core.CfnOutput(self,
"MeshArn",
value=self.mesh.mesh_arn,
export_name="MeshArn")
core.CfnOutput(self,
"MeshName",
value=self.mesh.mesh_name,
export_name="MeshName")
core.CfnOutput(
self,
"MeshEnvoyServiceArn",
value=self.mesh_gateway_proxy_fargate_service.service.service_arn,
export_name="MeshEnvoyServiceArn")
core.CfnOutput(self,
"MeshVGWArn",
value=self.mesh_vgw.virtual_gateway_arn,
export_name="MeshVGWArn")
core.CfnOutput(self,
"MeshVGWName",
value=self.mesh_vgw.virtual_gateway_name,
export_name="MeshVGWName")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# CONTAINER_IMAGE = 'daskdev/dask:0.19.4'
# if use_rapids:
# CONTAINER_IMAGE = 'rapidsai/rapidsai:latest'
# if use_notebook:
# CONTAINER_IMAGE = 'daskdev/dask-notebook:latest'
#TODO : Create ECR repository
#Update: Not required sunce ecs.ContainerImage already creates and pushes using same asset
#ecr = aws_ecr.Repository(self, 'MyECR', repository_name='dask')
# not needed if you use an asset like below:
dockercontainer = ecs.ContainerImage.from_asset(
directory='dockerstuff', build_args=['-t dask .'])
# Create vpc
vpc = ec2.Vpc(self, 'MyVpc', max_azs=3) # default is all AZs in region
subnets = vpc.private_subnets
# Create log groups for the scheduler and workers
s_logs = logs.LogGroup(self, 'SlogGroup', log_group_name='SlogGroup')
w_logs = logs.LogGroup(self, 'WlogGroup', log_group_name='WlogGroup')
#Create private namespace
#nspace = sd.PrivateDnsNamespace(self, 'MyNamespace', vpc=vpc, name='local-dask')
# #Create role for ECS
nRole = iam_.Role(self,
'ECSExecutionRole',
assumed_by=iam_.ServicePrincipal('ecs-tasks'))
nPolicy = iam_.Policy(
self,
"ECSExecutionPolicy",
policy_name="ECSExecutionPolicy",
statements=[
iam_.PolicyStatement(actions=[
'ecr:BatchCheckLayerAvailability',
'ecr:GetDownloadUrlForLayer', 'ecr:BatchGetImage',
'ecr:GetAuthorizationToken', 'logs:CreateLogStream',
'logs:PutLogEvents', 'sagemaker:*', 's3:*'
],
resources=[
'*',
]),
]).attach_to_role(nRole)
# Create ECS cluster
cluster = ecs.Cluster(self,
'DaskCluster',
vpc=vpc,
cluster_name='Fargate-Dask-Cluster')
nspace = cluster.add_default_cloud_map_namespace(
name='local-dask', type=sd.NamespaceType.DNS_PRIVATE, vpc=vpc)
#TO DO: Use default namespace for cluster and use cmap options within fargate service
#Update: done
# schedulerRegistry = sd.Service(self,'serviceRegistryScheduler',
# namespace=nspace,dns_ttl=core.Duration.seconds(60),
# custom_health_check=sd.HealthCheckCustomConfig(failure_threshold=10),
# name='Dask-Scheduler')
# # schedulerRegistry.register_ip_instance(id='serviceRegistryScheduler',ipv4='')
# workerRegistry = sd.Service(self,'workerRegistryScheduler',
# namespace=nspace,dns_ttl=core.Duration.seconds(60),
# custom_health_check=sd.HealthCheckCustomConfig(failure_threshold=10),
# name='Dask-Worker')
# -------------------- Add scheduler task ------------------------
schedulerTask = ecs.TaskDefinition(
self,
'taskDefinitionScheduler',
compatibility=ecs.Compatibility.FARGATE,
cpu='4096',
memory_mib='8192',
network_mode=ecs.NetworkMode.AWS_VPC,
placement_constraints=None,
execution_role=nRole,
family='Dask-Scheduler',
task_role=nRole)
schedulerTask.add_container('MySchedulerImage',
image=dockercontainer,
command=['dask-scheduler'],
cpu=4096,
essential=True,
logging=ecs.LogDriver.aws_logs(
stream_prefix='ecs', log_group=s_logs),
memory_limit_mib=8192,
memory_reservation_mib=8192)
# -------------------- Add worker task -----------------------------
workerTask = ecs.TaskDefinition(
self,
'taskDefinitionWorker',
compatibility=ecs.Compatibility.FARGATE,
cpu='4096',
memory_mib='8192',
network_mode=ecs.NetworkMode.AWS_VPC,
placement_constraints=None,
execution_role=nRole,
family='Dask-Worker',
task_role=nRole)
workerTask.add_container(
'MyWorkerImage',
image=dockercontainer,
command=[
'dask-worker', 'dask-scheduler.local-dask:8786',
'--memory-limit 1800MB', '--worker-port 9000',
'--nanny-port 9001', '--bokeh-port 9002'
],
cpu=4096,
essential=True,
logging=ecs.LogDriver.aws_logs(stream_prefix='ecs',
log_group=s_logs),
memory_limit_mib=8192,
memory_reservation_mib=8192)
# Task security group
sg = ec2.SecurityGroup(self,
'MySG',
vpc=vpc,
description='Enable Scheduler ports access',
security_group_name='DaskSecurityGroup')
# Ingress rule requires IPeer not Peer
# TO DO: fix from any ipv4 to SG
p1 = ec2.Peer().ipv4('0.0.0.0/0')
p2 = ec2.Peer().ipv4('0.0.0.0/0')
sg.add_ingress_rule(peer=p1,
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='p1',
from_port=8786,
to_port=8789))
sg.add_ingress_rule(peer=p2,
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='p2',
from_port=9000,
to_port=9002))
# ----------------- Add Scheduler Service -----------------------
# deployconfig = ecs.CfnService.DeploymentConfigurationProperty(maximum_percent=200,minimum_healthy_percent=100)
# vpcconfig = ecs.CfnService.AwsVpcConfigurationProperty(subnets = subnets,assign_public_ip=True,security_groups=[sg])
# networkconfig = ecs.CfnService.NetworkConfigurationProperty(awsvpc_configuration=vpcconfig)
# schedulerService = ecs.CfnService(self, 'DaskSchedulerService',
# task_definition = schedulerTask, deployment_configuration=deployconfig,
# cluster=cluster, desired_count=1, enable_ecs_managed_tags=None,
# launch_type='FARGATE',network_configuration=networkconfig,
# service_registries=schedulerRegistry)
#ecs.CfnService.ServiceRegistryProperty()
# Try fargate service? No service registry option available
#using default cluster namespace
cmap1 = ecs.CloudMapOptions(dns_ttl=core.Duration.seconds(60),
failure_threshold=10,
name='Dask-Scheduler')
schedulerService = ecs.FargateService(
self,
'DaskSchedulerService',
task_definition=schedulerTask,
assign_public_ip=True,
security_group=sg,
#vpc_subnets=subnets,
cluster=cluster,
desired_count=1,
max_healthy_percent=200,
min_healthy_percent=100,
service_name='Dask-Scheduler',
cloud_map_options=cmap1)
# schedulerService.enable_cloud_map(name = 'serviceRegistryScheduler')
# schedulerRegistry.register_non_ip_instance(self,instance_id='DaskSchedulerService')
# ----------------- Add Worker Service -----------------------
#using default cluster namespace
cmap2 = ecs.CloudMapOptions(dns_ttl=core.Duration.seconds(60),
failure_threshold=10,
name='Dask-Worker')
workerService = ecs.FargateService(
self,
'DaskWorkerService',
task_definition=workerTask,
assign_public_ip=True,
security_group=sg,
#vpc_subnets=subnets,
cluster=cluster,
desired_count=1,
max_healthy_percent=200,
min_healthy_percent=100,
service_name='Dask-Worker',
cloud_map_options=cmap2)
# workerService.enable_cloud_map(name = 'workerRegistryScheduler')
#------------------------------------------------------------------------
# Very less control with ECS patterns, did not work
# ecs_patterns.ApplicationLoadBalancedFargateService(self, "DaskFargateStack",
# cluster=cluster, # Required
# cpu=512, # Default is 256
# desired_count=6, # Default is 1
# task_image_options=ecs_patterns.ApplicationLoadBalancedTaskImageOptions(
# image=ecs.ContainerImage.from_registry(CONTAINER_IMAGE)),
# memory_limit_mib=2048, # Default is 512
# public_load_balancer=True) # Default is False
# Start a notebook in the same vpc
# print(type(sg.security_group_id))
# print("------------------------------")
# print(subnets[0].subnet_id)
#Create role for Notebook instance
smRole = iam_.Role(self,
"notebookAccessRole",
assumed_by=iam_.ServicePrincipal('sagemaker'))
smPolicy = iam_.Policy(self,
"notebookAccessPolicy",
policy_name="notebookAccessPolicy",
statements=[
iam_.PolicyStatement(
actions=['s3:*', 'ecs:*'],
resources=[
'*',
]),
]).attach_to_role(smRole)
notebook = sagemaker_.CfnNotebookInstance(
self,
'DaskNotebook',
instance_type='ml.t2.medium',
volume_size_in_gb=50,
security_group_ids=[sg.security_group_id],
subnet_id=subnets[0].subnet_id,
notebook_instance_name='DaskNotebook',
role_arn=smRole.role_arn,
root_access='Enabled',
direct_internet_access='Enabled',
default_code_repository=
'https://github.com/w601sxs/dask-examples.git')
def __init__(self,
scope: core.Construct,
id: str,
domain_name: str,
identity_provider_client_id: str,
identity_provider_client_secret: str,
identity_provider_client_url: str,
identity_provider_realm: str,
identity_provider_scope: str = 'openid',
vpc: ec2.IVpc = None,
cluster: ecs.ICluster = None,
load_balancer: elbv2.IApplicationLoadBalancer = None,
log_group: logs.ILogGroup = None,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
if vpc is None:
vpc = ec2.Vpc(self, 'ApplicationkVpc')
if cluster is None:
cluster = ecs.Cluster(self, 'ApplicationCluster', vpc=vpc)
if log_group is None:
log_group = logs.LogGroup(
self,
'ApplicationLogGroup',
retention=logs.RetentionDays.ONE_WEEK,
removal_policy=core.RemovalPolicy.DESTROY)
application_task_role = iam.Role(
self,
'ApplicationTaskRole',
assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'))
application_hosted_zone = route53.HostedZone.from_lookup(
self, 'ApplicationHostedZone', domain_name=domain_name)
application_certificate = acm.DnsValidatedCertificate(
self,
'FrontendAlbCertificate',
hosted_zone=application_hosted_zone,
domain_name='app.' + domain_name)
application_service = ecs_patterns.ApplicationLoadBalancedFargateService(
self,
'ApplicationLoadBalancedFargateService',
cluster=cluster,
load_balancer=load_balancer,
task_image_options=ecs_patterns.
ApplicationLoadBalancedTaskImageOptions(
image=ecs.ContainerImage.from_asset("application"),
enable_logging=True,
log_driver=ecs.AwsLogDriver(stream_prefix='application',
log_group=log_group),
task_role=application_task_role,
container_port=8080,
),
memory_limit_mib=512,
cpu=256,
desired_count=1,
public_load_balancer=True,
domain_name='app.' + domain_name,
domain_zone=application_hosted_zone,
protocol=elbv2.ApplicationProtocol.HTTPS,
)
application_service.target_group.enable_cookie_stickiness(
core.Duration.seconds(24 * 60 * 60))
application_service.target_group.configure_health_check(
port='8080',
path='/',
timeout=core.Duration.seconds(20),
healthy_threshold_count=2,
unhealthy_threshold_count=10,
interval=core.Duration.seconds(30),
)
application_service.listener.add_certificates(
'ApplicationServiceCertificate',
certificates=[application_certificate])
application_service.listener.add_action(
'DefaultAction',
action=elbv2.ListenerAction.authenticate_oidc(
authorization_endpoint=identity_provider_client_url +
'/auth/realms/' + identity_provider_realm +
'/protocol/openid-connect/auth',
token_endpoint=identity_provider_client_url + '/auth/realms/' +
identity_provider_realm + '/protocol/openid-connect/token',
user_info_endpoint=identity_provider_client_url +
'/auth/realms/' + identity_provider_realm +
'/protocol/openid-connect/userinfo',
issuer=identity_provider_client_url + '/auth/realms/' +
identity_provider_realm,
client_id=identity_provider_client_id,
client_secret=core.SecretValue(
identity_provider_client_secret),
scope=identity_provider_scope,
on_unauthenticated_request=elbv2.UnauthenticatedAction.
AUTHENTICATE,
next=elbv2.ListenerAction.forward(
[application_service.target_group]),
))
application_service.load_balancer.connections.allow_to_any_ipv4(
port_range=ec2.Port(
from_port=443,
to_port=443,
protocol=ec2.Protocol.TCP,
string_representation='Allow ALB to verify token'))
def __init__(
self,
scope: core.Construct,
id: str,
cpu: Union[int, float] = 256,
memory: Union[int, float] = 512,
mincount: int = 1,
maxcount: int = 50,
permissions: Optional[List[iam.PolicyStatement]] = None,
code_dir: str = "./titiler/",
**kwargs: Any,
) -> None:
"""Define stack."""
super().__init__(scope, id, **kwargs)
permissions = permissions or []
vpc = ec2.Vpc(self, f"{id}-vpc", max_azs=2)
cluster = ecs.Cluster(self, f"{id}-cluster", vpc=vpc)
task_env = DEFAULT_ENV.copy()
task_env.update(
dict(
MODULE_NAME="titiler.main",
VARIABLE_NAME="app",
LOG_LEVEL="error",
))
# GUNICORN configuration
if settings.workers_per_core:
task_env.update(
{"WORKERS_PER_CORE": str(settings.workers_per_core)})
if settings.max_workers:
task_env.update({"MAX_WORKERS": str(settings.max_workers)})
if settings.web_concurrency:
task_env.update({"WEB_CONCURRENCY": str(settings.web_concurrency)})
fargate_service = ecs_patterns.ApplicationLoadBalancedFargateService(
self,
f"{id}-service",
cluster=cluster,
cpu=cpu,
memory_limit_mib=memory,
desired_count=mincount,
public_load_balancer=True,
listener_port=80,
task_image_options=dict(
image=ecs.ContainerImage.from_asset(
code_dir,
exclude=["cdk.out", ".git"],
file="Dockerfiles/ecs/Dockerfile",
),
container_port=80,
environment=task_env,
),
)
for perm in permissions:
fargate_service.task_definition.task_role.add_to_policy(perm)
scalable_target = fargate_service.service.auto_scale_task_count(
min_capacity=mincount, max_capacity=maxcount)
# https://github.com/awslabs/aws-rails-provisioner/blob/263782a4250ca1820082bfb059b163a0f2130d02/lib/aws-rails-provisioner/scaling.rb#L343-L387
scalable_target.scale_on_request_count(
"RequestScaling",
requests_per_target=50,
scale_in_cooldown=core.Duration.seconds(240),
scale_out_cooldown=core.Duration.seconds(30),
target_group=fargate_service.target_group,
)
# scalable_target.scale_on_cpu_utilization(
# "CpuScaling", target_utilization_percent=70,
# )
fargate_service.service.connections.allow_from_any_ipv4(
port_range=ec2.Port(
protocol=ec2.Protocol.ALL,
string_representation="All port 80",
from_port=80,
),
description="Allows traffic on port 80 from NLB",
)
#docker_volume=ecs.Volume(name='docker_vol',docker_volume_configuration=docker_vol_config)
#efs_mount=ecs.MountPoint(container_path='/efs',read_only=True, source_volume='docker_vol')
cluster = ecs.Cluster(stack,
"wes-ecs",
vpc=vpc,
cluster_name='Proj-VONC_VISTA')
print('cluster sec group ', str(type(cluster.autoscaling_group)))
#cluster.add_capacity("DefaultAutoScalingGroup",
# instance_type=ec2.InstanceType("c5.xlarge"), key_name='Vonc-Prod-Key',max_capacity=4,machine_image=amitouse,
# desired_capacity=2,min_capacity=2)
print('connections ', str(cluster.connections))
port = ec2.Port(protocol=ec2.Protocol.TCP,
string_representation='inbound to container instances',
from_port=22,
to_port=22)
cluster.connections.add_security_group(app_security_group_import)
cluster.connections.allow_from_any_ipv4(port,
'in bound to container instances')
# Create a task definition with its own elastic network interface
task_definition_vistaweb = ecs.FargateTaskDefinition(
stack,
"Proj-VONC_VISTA-task-vistaweb",
memory_limit_mib=8192,
cpu=4096,
family='Proj-VONC_VISTA-Vista')
task_definition_vistaweb.add_to_execution_role_policy(custom_policy)
task_definition_vistaweb.add_to_task_role_policy(custom_policy)
def provision_bastion(self, name: str, bastion: VPC.Bastion) -> None:
if not bastion.enabled:
return None
if bastion.machine_image:
bastion_machine_image = ec2.MachineImage.generic_linux(
{self.region: bastion.machine_image.ami_id},
user_data=ec2.UserData.custom(bastion.machine_image.user_data),
)
else:
if not self.scope.account.isnumeric(
): # TODO: Can we get rid of this requirement?
raise ValueError(
"Error loooking up AMI: Must provide explicit AWS account ID to do AMI lookup. Either provide AMI ID or AWS account id"
)
bastion_machine_image = ec2.LookupMachineImage(
name="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*",
owners=["099720109477"])
bastion_sg = ec2.SecurityGroup(
self.scope,
"bastion_sg",
vpc=self.vpc,
security_group_name=f"{name}-bastion",
)
for rule in bastion.ingress_ports:
for ip_cidr in rule.ip_cidrs:
bastion_sg.add_ingress_rule(
peer=ec2.Peer.ipv4(ip_cidr),
connection=ec2.Port(
protocol=ec2.Protocol(rule.protocol),
string_representation=rule.name,
from_port=rule.from_port,
to_port=rule.to_port,
),
)
bastion = ec2.Instance(
self.scope,
"bastion",
machine_image=bastion_machine_image,
vpc=self.vpc,
instance_type=ec2.InstanceType(bastion.instance_type),
key_name=bastion.key_name,
security_group=bastion_sg,
vpc_subnets=ec2.SubnetSelection(
subnet_group_name=self.public_subnet_name, ),
)
ec2.CfnEIP(
self.scope,
"bastion_eip",
instance_id=bastion.instance_id,
)
cdk.CfnOutput(self.scope,
"bastion_public_ip",
value=bastion.instance_public_ip)
return bastion_sg
def __init__(self, scope: core.Construct, id: str,
landing_zone: ILandingZone,
directory: DirectoryServicesConstruct, group_names: [List],
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.__landing_zone = landing_zone
# Configure the security groups
self.security_group = ec2.SecurityGroup(
self,
'SecurityGroup',
vpc=landing_zone.networking.vpc,
allow_all_outbound=True,
description='HadoopConstruct Security Group',
security_group_name='hadoop-mapreduce-group')
for port in services.keys():
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
from_port=port,
to_port=port,
string_representation=services[port]))
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.UDP,
from_port=0,
to_port=65535,
string_representation='Allow All UDP Traffic'))
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
from_port=0,
to_port=65535,
string_representation='Allow All TCP Traffic'))
# Setup roles...
self.jobFlowRole = iam.Role(
self,
'JobFlowRole',
assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceforEC2Role'),
])
profile_name = 'jobflowprofile@{}-{}'.format(
landing_zone.zone_name,
core.Stack.of(self).region)
job_flow_instance_profile = iam.CfnInstanceProfile(
self,
'JobFlowInstanceProfile',
instance_profile_name=profile_name,
roles=[self.jobFlowRole.role_name])
serviceRole = iam.Role(
self,
'ServiceRole',
assumed_by=iam.ServicePrincipal(
service='elasticmapreduce.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceRole')
])
self.database = g.Database(self,
'GlueStore',
database_name='demo-database')
self.bucket = s3.Bucket(self,
'LogBucket',
removal_policy=core.RemovalPolicy.DESTROY)
emr_fs = EmrfsConstruct(self,
'Emrfs',
landing_zone=landing_zone,
directory=directory,
group_names=group_names,
job_flow_role=self.jobFlowRole)
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticmapreduce-instancefleetconfig.html
self.cluster = emr.CfnCluster(
self,
'Hadoop',
name='HadoopCluster',
job_flow_role=profile_name, #'EMR_EC2_DefaultRole',
service_role=serviceRole.role_name,
log_uri='s3://' + self.bucket.bucket_name + '/logs',
release_label='emr-6.2.0',
applications=[
emr.CfnCluster.ApplicationProperty(name='Spark'),
emr.CfnCluster.ApplicationProperty(name='Presto'),
emr.CfnCluster.ApplicationProperty(name='Hue'),
emr.CfnCluster.ApplicationProperty(name='Hive'),
emr.CfnCluster.ApplicationProperty(name='JupyterHub'),
],
configurations=[
emr.CfnCluster.ConfigurationProperty(
classification='spark-hive-site',
configuration_properties={
'hive.metastore.client.factory.class':
'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory'
}),
emr.CfnCluster.ConfigurationProperty(
classification='hive-site',
configuration_properties={
'hive.metastore.client.factory.class':
'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory',
'aws.glue.partition.num.segments':
'10', #1 to 10; (default=5)
'hive.metastore.schema.verification': 'false',
})
],
security_configuration=emr_fs.security_configuration.ref,
# kerberos_attributes= emr.CfnCluster.KerberosAttributesProperty(
# kdc_admin_password=directory.password,
# realm= directory.mad.name.upper(),
# ad_domain_join_password=directory.password,
# ad_domain_join_user= directory.admin
# ),
managed_scaling_policy=emr.CfnCluster.ManagedScalingPolicyProperty(
compute_limits=emr.CfnCluster.ComputeLimitsProperty(
minimum_capacity_units=1,
maximum_capacity_units=25,
unit_type='InstanceFleetUnits')),
instances=emr.CfnCluster.JobFlowInstancesConfigProperty(
#hadoop_version='2.4.0',
termination_protected=False,
master_instance_fleet=emr.CfnCluster.
InstanceFleetConfigProperty(
target_spot_capacity=1,
instance_type_configs=[
emr.CfnCluster.InstanceTypeConfigProperty(
instance_type='m5.xlarge', )
]),
core_instance_fleet=emr.CfnCluster.InstanceFleetConfigProperty(
target_spot_capacity=1,
instance_type_configs=[
emr.CfnCluster.InstanceTypeConfigProperty(
instance_type='m5.xlarge',
ebs_configuration=emr.CfnCluster.
EbsConfigurationProperty(ebs_block_device_configs=[
emr.CfnCluster.EbsBlockDeviceConfigProperty(
volume_specification=emr.CfnCluster.
VolumeSpecificationProperty(
size_in_gb=50, volume_type='gp2'))
]))
]),
additional_master_security_groups=[
self.security_group.security_group_id
],
additional_slave_security_groups=[
self.security_group.security_group_id
],
ec2_subnet_ids=[
net.subnet_id for net in landing_zone.networking.vpc.
_select_subnet_objects(subnet_group_name='Hadoop')
],
))
self.cluster.add_depends_on(job_flow_instance_profile)
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc,
region: str) -> None:
super().__init__(scope, id)
# create an IAM role to attach to the squid instances
squid_iam_role = iam.Role(
self,
"squid-role",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"CloudWatchAgentServerPolicy"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2RoleforSSM")
])
# Add policy to allow EC2 update instance attributes
squid_iam_role.add_to_policy(
statement=iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=[
'ec2:ModifyInstanceAttribute',
],
resources=['*']))
# Create bucket to hold Squid config and whitelist files
squid_config_bucket = s3.Bucket(
self, "squid-config", encryption=s3.BucketEncryption.KMS_MANAGED)
# Upload config and whiteliest files to S3 bucket
s3_deployment.BucketDeployment(
self,
"config",
destination_bucket=squid_config_bucket,
sources=[
s3_deployment.Source.asset(
path='./squid_app/squid_config_files/config_files_s3')
])
# Provide access to EC2 instance role to read and write to bucket
squid_config_bucket.grant_read_write(identity=squid_iam_role)
# Set the AMI to the latest Amazon Linux 2
amazon_linux_2_ami = ec2.MachineImage.latest_amazon_linux(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=ec2.AmazonLinuxEdition.STANDARD,
virtualization=ec2.AmazonLinuxVirt.HVM,
storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE)
if vpc.public_subnets:
# Squid ASGs with desired capacity as 1 Instance in each of the AZs
self.squid_asgs = []
for count, az in enumerate(vpc.availability_zones, start=1):
asg = autoscaling.AutoScalingGroup(
self,
f"asg-{count}",
vpc=vpc,
instance_type=ec2.InstanceType("t3.nano"),
desired_capacity=1,
max_capacity=1,
min_capacity=1,
machine_image=amazon_linux_2_ami,
role=squid_iam_role,
vpc_subnets=ec2.SubnetSelection(
availability_zones=[az],
one_per_az=True,
subnet_type=ec2.SubnetType.PUBLIC),
health_check=autoscaling.HealthCheck.ec2(
grace=core.Duration.minutes(5)),
resource_signal_count=1,
resource_signal_timeout=core.Duration.minutes(10))
cfn_asg: autoscaling.CfnAutoScalingGroup = asg.node.default_child
asg_logical_id = cfn_asg.logical_id
# User data: Required parameters in user data script
user_data_mappings = {
"__S3BUCKET__": squid_config_bucket.bucket_name,
"__ASG__": asg_logical_id,
"__CW_ASG__": "${aws:AutoScalingGroupName}"
}
# Replace parameters with values in the user data
with open(
"./squid_app/squid_config_files/user_data/squid_user_data.sh",
'r') as user_data_h:
# Use a substitution
user_data_sub = core.Fn.sub(user_data_h.read(),
user_data_mappings)
# Add User data to Launch Config of the autoscaling group
asg.add_user_data(user_data_sub)
# Security group attached to the ASG Squid instances
# Outbound: All allowed
# Inboud: Allowed from VPC CIDR on ports 80, 443)
asg.connections.allow_from(
other=ec2.Peer.ipv4(vpc.vpc_cidr_block),
port_range=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="HTTP from VPC",
from_port=80,
to_port=80))
asg.connections.allow_from(
other=ec2.Peer.ipv4(vpc.vpc_cidr_block),
port_range=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="HTTPS from VPC",
from_port=443,
to_port=443))
# Create ASG Lifecycle hook to enable updating of route table using Lambda when instance launches and is marked Healthy
autoscaling.LifecycleHook(
self,
f"asg-hook-{count}",
auto_scaling_group=asg,
lifecycle_transition=autoscaling.LifecycleTransition.
INSTANCE_LAUNCHING,
notification_target=hooktargets.TopicHook(
sns.Topic(self,
f"squid-asg-{count}-lifecycle-hook-topic",
display_name=
f"Squid ASG {count} Lifecycle Hook topic")),
default_result=autoscaling.DefaultResult.ABANDON,
heartbeat_timeout=core.Duration.minutes(5))
# Tag ASG with the route table IDs used by the isolated and/or private subnets in the availability zone
# This tag will be used by the Squid Lambda function to identify route tables to update when alarm changes from ALARM to OK
private_subnets_in_az = []
isolated_subnets_in_az = []
route_table_ids = ''
if vpc.private_subnets:
private_subnets_in_az = vpc.select_subnets(
availability_zones=[az],
subnet_type=ec2.SubnetType.PRIVATE).subnets
if vpc.isolated_subnets:
isolated_subnets_in_az = vpc.select_subnets(
availability_zones=[az],
subnet_type=ec2.SubnetType.ISOLATED).subnets
non_public_subnets_in_az = isolated_subnets_in_az + private_subnets_in_az
# Loop through all non public subnets in AZ to identify route table and create a tag value string
for subnet in non_public_subnets_in_az:
if route_table_ids:
route_table_ids = f"{route_table_ids},{subnet.route_table.route_table_id}"
else:
route_table_ids = subnet.route_table.route_table_id
# Tag the ASG with route table ids
core.Tag.add(asg,
key='RouteTableIds',
value=route_table_ids,
apply_to_launched_instances=False)
self.squid_asgs.append(asg)
else:
raise ValueError("No public subnets in VPC")
def __init__(self, scope: core.Stack, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.environment_name = 'ecsworkshop'
## Importing existing VPC and ECS Cluster ##
self.vpc = ec2.Vpc.from_lookup(self,
"VPC",
vpc_name='{}-base/BaseVPC'.format(
self.environment_name))
self.sd_namespace = sd.PrivateDnsNamespace.from_private_dns_namespace_attributes(
self,
"SDNamespace",
namespace_name=core.Fn.import_value('NSNAME'),
namespace_arn=core.Fn.import_value('NSARN'),
namespace_id=core.Fn.import_value('NSID'))
self.ecs_cluster = ecs.Cluster.from_cluster_attributes(
self,
"ECSCluster",
cluster_name=core.Fn.import_value('ECSClusterName'),
security_groups=[],
vpc=self.vpc,
default_cloud_map_namespace=self.sd_namespace)
## End VPC and ECS Cluster ##
## Load balancer for ECS service ##
self.frontend_sec_grp = ec2.SecurityGroup(
self,
"FrontendIngress",
vpc=self.vpc,
allow_all_outbound=True,
description="Frontend Ingress All port 80",
)
self.load_balancer = elbv2.ApplicationLoadBalancer(
self,
"ALB",
security_group=self.frontend_sec_grp,
internet_facing=True,
vpc=self.vpc)
self.target_group = elbv2.ApplicationTargetGroup(
self,
"ALBTG",
port=8000,
target_group_name="ECSDemoFargateEFS",
vpc=self.vpc,
target_type=elbv2.TargetType.IP)
self.load_balancer.add_listener(
"FrontendListener",
default_target_groups=[self.target_group],
port=80)
## End Load balancer ##
## EFS Setup ##
self.service_sec_grp = ec2.SecurityGroup(
self,
"EFSSecGrp",
vpc=self.vpc,
description="Allow access to self on NFS Port",
)
self.service_sec_grp.connections.allow_from(
other=self.service_sec_grp,
port_range=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="Self",
from_port=2049,
to_port=2049))
# TODO: possibly create another sec grp for 8000
self.service_sec_grp.connections.allow_from(
other=self.frontend_sec_grp,
port_range=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="LB2Service",
from_port=8000,
to_port=8000))
self.shared_fs = efs.FileSystem(
self,
"SharedFS",
vpc=self.vpc,
security_group=self.service_sec_grp,
)
## End EFS Setup ##
## TODO: IAM Role to access EFS access points for task ##
# Task execution role
self.task_execution_role = iam.Role(
self,
"TaskExecutionRole",
assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'),
description="Task execution role for ecs services",
managed_policies=[
iam.ManagedPolicy.from_managed_policy_arn(
self,
'arn',
managed_policy_arn=
'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy'
)
])
## END IAM ##
## Logging ##
self.service_log_group = logs.LogGroup(self, "ECSEFSDemoLogGrp")
## END Logging ##
# Cloudformation Outputs
core.CfnOutput(self,
"ExecutionRoleARN",
value=self.task_execution_role.role_arn,
export_name="ECSFargateEFSDemoTaskExecutionRoleARN")
core.CfnOutput(self,
"EFSID",
value=self.shared_fs.file_system_id,
export_name="ECSFargateEFSDemoFSID")
core.CfnOutput(self,
"LBName",
value=self.load_balancer.load_balancer_name,
export_name="ECSFargateEFSDemoLBName")
core.CfnOutput(self,
"TargetGroupArn",
value=self.target_group.target_group_arn,
export_name="ECSFargateEFSDemoTGARN")
core.CfnOutput(self,
"VPCPrivateSubnets",
value=",".join(
[x.subnet_id for x in self.vpc.private_subnets]),
export_name="ECSFargateEFSDemoPrivSubnets")
core.CfnOutput(self,
"SecurityGroups",
value="{},{}".format(
self.frontend_sec_grp.security_group_id,
self.service_sec_grp.security_group_id),
export_name="ECSFargateEFSDemoSecGrps")
core.CfnOutput(self,
"LBURL",
value=self.load_balancer.load_balancer_dns_name,
export_name="ECSFargateEFSDemoLBURL")
core.CfnOutput(self,
"LogGroupName",
value=self.service_log_group.log_group_name,
export_name="ECSFargateEFSDemoLogGroupName")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
eks_vpc = ec2.Vpc(self, "VPC", cidr="10.0.0.0/16")
self.eks_vpc = eks_vpc
# Create IAM Role For code-server bastion
bastion_role = iam.Role(
self,
"BastionRole",
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal("ec2.amazonaws.com"),
iam.AccountRootPrincipal()),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AdministratorAccess")
])
self.bastion_role = bastion_role
# Create EC2 Instance Profile for that Role
instance_profile = iam.CfnInstanceProfile(
self, "InstanceProfile", roles=[bastion_role.role_name])
# Create SecurityGroup for the Control Plane ENIs
eks_security_group = ec2.SecurityGroup(self,
"EKSSecurityGroup",
vpc=eks_vpc,
allow_all_outbound=True)
eks_security_group.add_ingress_rule(ec2.Peer.ipv4('10.0.0.0/16'),
ec2.Port.all_traffic())
# Create an EKS Cluster
eks_cluster = eks.Cluster(
self,
"cluster",
cluster_name="cluster",
vpc=eks_vpc,
masters_role=bastion_role,
default_capacity_type=eks.DefaultCapacityType.NODEGROUP,
default_capacity_instance=ec2.InstanceType("m5.large"),
default_capacity=2,
security_group=eks_security_group,
endpoint_access=eks.EndpointAccess.PUBLIC_AND_PRIVATE,
version=eks.KubernetesVersion.V1_17)
self.cluster_cert = eks_cluster.cluster_certificate_authority_data
# Deploy ALB Ingress Controller
# Create the k8s Service account and corresponding IAM Role mapped via IRSA
alb_service_account = eks_cluster.add_service_account(
"alb-ingress-controller",
name="alb-ingress-controller",
namespace="kube-system")
# Create the PolicyStatements to attach to the role
# I couldn't find a way to get this to work with a PolicyDocument and there are 10 of these
alb_policy_statement_json_1 = {
"Effect":
"Allow",
"Action": [
"acm:DescribeCertificate", "acm:ListCertificates",
"acm:GetCertificate"
],
"Resource":
"*"
}
alb_policy_statement_json_2 = {
"Effect":
"Allow",
"Action": [
"ec2:AuthorizeSecurityGroupIngress", "ec2:CreateSecurityGroup",
"ec2:CreateTags", "ec2:DeleteTags", "ec2:DeleteSecurityGroup",
"ec2:DescribeAccountAttributes", "ec2:DescribeAddresses",
"ec2:DescribeInstances", "ec2:DescribeInstanceStatus",
"ec2:DescribeInternetGateways",
"ec2:DescribeNetworkInterfaces", "ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVpcs",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:RevokeSecurityGroupIngress"
],
"Resource":
"*"
}
alb_policy_statement_json_3 = {
"Effect":
"Allow",
"Action": [
"elasticloadbalancing:AddListenerCertificates",
"elasticloadbalancing:AddTags",
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:CreateRule",
"elasticloadbalancing:CreateTargetGroup",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:DeleteRule",
"elasticloadbalancing:DeleteTargetGroup",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:DescribeListenerCertificates",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"elasticloadbalancing:DescribeRules",
"elasticloadbalancing:DescribeSSLPolicies",
"elasticloadbalancing:DescribeTags",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetGroupAttributes",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:ModifyRule",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:ModifyTargetGroupAttributes",
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:RemoveListenerCertificates",
"elasticloadbalancing:RemoveTags",
"elasticloadbalancing:SetIpAddressType",
"elasticloadbalancing:SetSecurityGroups",
"elasticloadbalancing:SetSubnets",
"elasticloadbalancing:SetWebAcl"
],
"Resource":
"*"
}
alb_policy_statement_json_4 = {
"Effect":
"Allow",
"Action": [
"iam:CreateServiceLinkedRole", "iam:GetServerCertificate",
"iam:ListServerCertificates"
],
"Resource":
"*"
}
alb_policy_statement_json_5 = {
"Effect": "Allow",
"Action": ["cognito-idp:DescribeUserPoolClient"],
"Resource": "*"
}
alb_policy_statement_json_6 = {
"Effect":
"Allow",
"Action": [
"waf-regional:GetWebACLForResource", "waf-regional:GetWebACL",
"waf-regional:AssociateWebACL",
"waf-regional:DisassociateWebACL"
],
"Resource":
"*"
}
alb_policy_statement_json_7 = {
"Effect": "Allow",
"Action": ["tag:GetResources", "tag:TagResources"],
"Resource": "*"
}
alb_policy_statement_json_8 = {
"Effect": "Allow",
"Action": ["waf:GetWebACL"],
"Resource": "*"
}
alb_policy_statement_json_9 = {
"Effect":
"Allow",
"Action": [
"wafv2:GetWebACL", "wafv2:GetWebACLForResource",
"wafv2:AssociateWebACL", "wafv2:DisassociateWebACL"
],
"Resource":
"*"
}
alb_policy_statement_json_10 = {
"Effect":
"Allow",
"Action": [
"shield:DescribeProtection", "shield:GetSubscriptionState",
"shield:DeleteProtection", "shield:CreateProtection",
"shield:DescribeSubscription", "shield:ListProtections"
],
"Resource":
"*"
}
# Attach the necessary permissions
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_1))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_2))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_3))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_4))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_5))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_6))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_7))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_8))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_9))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_10))
# Deploy the ALB Ingress Controller from the Helm chart
eks_cluster.add_helm_chart(
"aws-alb-ingress-controller",
chart="aws-alb-ingress-controller",
repository=
"http://storage.googleapis.com/kubernetes-charts-incubator",
namespace="kube-system",
values={
"clusterName": "cluster",
"awsRegion": os.environ["CDK_DEFAULT_REGION"],
"awsVpcID": eks_vpc.vpc_id,
"rbac": {
"create": True,
"serviceAccount": {
"create": False,
"name": "alb-ingress-controller"
}
}
})
# Create code-server bastion
# Get Latest Amazon Linux AMI
amzn_linux = ec2.MachineImage.latest_amazon_linux(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=ec2.AmazonLinuxEdition.STANDARD,
virtualization=ec2.AmazonLinuxVirt.HVM,
storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE)
# Create SecurityGroup for code-server
security_group = ec2.SecurityGroup(self,
"SecurityGroup",
vpc=eks_vpc,
allow_all_outbound=True)
security_group.add_ingress_rule(ec2.Peer.any_ipv4(),
ec2.Port.tcp(8080))
# Create our EC2 instance running CodeServer
code_server_instance = ec2.Instance(
self,
"CodeServerInstance",
instance_type=ec2.InstanceType("t3.large"),
machine_image=amzn_linux,
role=bastion_role,
vpc=eks_vpc,
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
security_group=security_group,
block_devices=[
ec2.BlockDevice(device_name="/dev/xvda",
volume=ec2.BlockDeviceVolume.ebs(20))
])
# Add UserData
code_server_instance.user_data.add_commands(
"mkdir -p ~/.local/lib ~/.local/bin ~/.config/code-server")
code_server_instance.user_data.add_commands(
"curl -fL https://github.com/cdr/code-server/releases/download/v3.5.0/code-server-3.5.0-linux-amd64.tar.gz | tar -C ~/.local/lib -xz"
)
code_server_instance.user_data.add_commands(
"mv ~/.local/lib/code-server-3.5.0-linux-amd64 ~/.local/lib/code-server-3.5.0"
)
code_server_instance.user_data.add_commands(
"ln -s ~/.local/lib/code-server-3.5.0/bin/code-server ~/.local/bin/code-server"
)
code_server_instance.user_data.add_commands(
"echo \"bind-addr: 0.0.0.0:8080\" > ~/.config/code-server/config.yaml"
)
code_server_instance.user_data.add_commands(
"echo \"auth: password\" >> ~/.config/code-server/config.yaml")
code_server_instance.user_data.add_commands(
"echo \"password: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)\" >> ~/.config/code-server/config.yaml"
)
code_server_instance.user_data.add_commands(
"echo \"cert: false\" >> ~/.config/code-server/config.yaml")
code_server_instance.user_data.add_commands(
"~/.local/bin/code-server &")
code_server_instance.user_data.add_commands(
"yum -y install jq gettext bash-completion moreutils")
code_server_instance.user_data.add_commands(
"sudo pip install --upgrade awscli && hash -r")
code_server_instance.user_data.add_commands(
"echo 'export ALB_INGRESS_VERSION=\"v1.1.8\"' >> ~/.bash_profile")
code_server_instance.user_data.add_commands(
"curl --silent --location -o /usr/local/bin/kubectl \"https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.9/2020-08-04/bin/linux/amd64/kubectl\""
)
code_server_instance.user_data.add_commands(
"chmod +x /usr/local/bin/kubectl")
code_server_instance.user_data.add_commands(
"curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash"
)
code_server_instance.user_data.add_commands(
"export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account)"
)
code_server_instance.user_data.add_commands(
"export AWS_REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r '.region')"
)
code_server_instance.user_data.add_commands(
"echo \"export ACCOUNT_ID=${ACCOUNT_ID}\" | tee -a ~/.bash_profile"
)
code_server_instance.user_data.add_commands(
"echo \"export AWS_REGION=${AWS_REGION}\" | tee -a ~/.bash_profile"
)
code_server_instance.user_data.add_commands(
"aws configure set default.region ${AWS_REGION}")
code_server_instance.user_data.add_commands(
"curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"
)
code_server_instance.user_data.add_commands("yum -y install nodejs")
code_server_instance.user_data.add_commands(
"amazon-linux-extras enable python3")
code_server_instance.user_data.add_commands(
"yum install -y python3 --disablerepo amzn2-core")
code_server_instance.user_data.add_commands("yum install -y git")
code_server_instance.user_data.add_commands(
"rm /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python && ln -s /usr/bin/pip3 /usr/bin/pip"
)
code_server_instance.user_data.add_commands("npm install -g aws-cdk")
code_server_instance.user_data.add_commands(
"echo 'export KUBECONFIG=~/.kube/config' >> ~/.bash_profile")
code_server_instance.user_data.add_commands(
"git clone https://github.com/jasonumiker/eks-school.git")
# Add ALB
lb = elbv2.ApplicationLoadBalancer(self,
"LB",
vpc=eks_vpc,
internet_facing=True)
listener = lb.add_listener("Listener", port=80)
listener.connections.allow_default_port_from_any_ipv4(
"Open to the Internet")
listener.connections.allow_to_any_ipv4(
port_range=ec2.Port(string_representation="TCP 8080",
protocol=ec2.Protocol.TCP,
from_port=8080,
to_port=8080))
listener.add_targets(
"Target",
port=8080,
targets=[
elbv2.InstanceTarget(
instance_id=code_server_instance.instance_id, port=8080)
])
