def create_instance_profile(self, job_role: iam.Role) -> iam.CfnInstanceProfile:
return iam.CfnInstanceProfile(
self,
"emr_instance_profile",
instance_profile_name=job_role.role_name,
roles=[job_role.role_name],
)
def add_instance_profile(self):
iam.CfnInstanceProfile(
self,
id=f"iam-{self.deploy_env.value}-glue-data-lake-{self.layer.value}-instance-profile",
instance_profile_name=f"iam-{self.deploy_env.value}-glue-data-lake-{self.layer.value}-instance-profile",
roles=[self.role_name],
)
def _add_instance_profile(self,
cleanup_policy_statements,
instance_role=None):
"""Set default instance profile in imagebuilder cfn template."""
instance_profile_resource = iam.CfnInstanceProfile(
self,
"InstanceProfile",
path=IAM_ROLE_PATH,
roles=[
instance_role.split("/")[-1]
if instance_role else Fn.ref("InstanceRole")
],
instance_profile_name=self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
)
if not self.custom_cleanup_lambda_role:
self._add_resource_delete_policy(
cleanup_policy_statements,
["iam:DeleteInstanceProfile"],
[
self.format_arn(
service="iam",
region="",
resource="instance-profile",
resource_name="{0}/{1}".format(
IAM_ROLE_PATH.strip("/"),
self._build_resource_name(
IMAGEBUILDER_RESOURCE_NAME_PREFIX),
),
)
],
)
return instance_profile_resource
def _add_ecs_instance_role_and_profile(self):
ecs_instance_role = iam.CfnRole(
self.stack_scope,
"EcsInstanceRole",
path=self._cluster_scoped_iam_path(),
managed_policy_arns=[
self._format_arn(
service="iam",
account="aws",
region="",
resource=
"policy/service-role/AmazonEC2ContainerServiceforEC2Role",
)
],
assume_role_policy_document=get_assume_role_policy_document(
f"ec2.{self._url_suffix}"),
)
iam_instance_profile = iam.CfnInstanceProfile(
self.stack_scope,
"IamInstanceProfile",
path=self._cluster_scoped_iam_path(),
roles=[ecs_instance_role.ref])
return ecs_instance_role, iam_instance_profile
def add_instance_profile(self):
instance_profile = iam.CfnInstanceProfile(
self,
id=f'iam-{self.environment}-glue-data-lake-instance-profile',
instance_profile_name=
f'iam-{self.environment}-glue-data-lake-instance-profile',
roles=[self.role_name])
return instance_profile
def __init__(
self,
scope: core.Construct,
common_stack: CommonResourcesStack,
**kwargs,
) -> None:
self.deploy_env = active_environment
self.s3_script_bucket = common_stack.s3_script_bucket
self.s3_log_bucket = common_stack.s3_log_bucket
super().__init__(scope,
id=f"{self.deploy_env.value}-emr-stack",
**kwargs)
# enable reading scripts from s3 bucket
self.read_scripts_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"s3:GetObject",
],
resources=[f"arn:aws:s3:::{self.s3_script_bucket.bucket_name}/*"],
)
self.read_scripts_document = iam.PolicyDocument()
self.read_scripts_document.add_statements(self.read_scripts_policy)
# emr service role
self.emr_service_role = iam.Role(
self,
"emr_service_role",
assumed_by=iam.ServicePrincipal("elasticmapreduce.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonElasticMapReduceRole")
],
inline_policies=[self.read_scripts_document],
)
# emr job flow role
self.emr_job_flow_role = iam.Role(
self,
"emr_job_flow_role",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonElasticMapReduceforEC2Role")
],
)
# emr job flow profile
self.emr_job_flow_profile = iam.CfnInstanceProfile(
self,
"emr_job_flow_profile",
roles=[self.emr_job_flow_role.role_name],
instance_profile_name="emrJobFlowProfile",
)
def __init__(self, scope: core.Construct, id: str, app_nw_stack: appNwStack, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Contexts
PRJ = self.node.try_get_context("prj")
AMI_ID = self.node.try_get_context("ami-id") # Amazon Linux 2
# Functions
def nametag(x): return core.CfnTag(
key="Name", value="{}/{}".format(PRJ, x))
# ### Resources
# IAM Role
ec2_statement = iam.PolicyStatement()
ec2_statement.add_actions("sts:AssumeRole")
ec2_statement.add_service_principal(service="ec2.amazonaws.com")
ec2_document = iam.PolicyDocument(
statements=[ec2_statement]
)
iam_role = iam.CfnRole(
self, "iamRoleForEc2",
role_name="{}-ec2-role".format(PRJ),
description="{}-ec2-role".format(PRJ),
assume_role_policy_document=ec2_document.to_json(),
managed_policy_arns=[
"arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM"
]
)
ec2_instance_profile = iam.CfnInstanceProfile(
self, "ec2InstanceProfile",
roles=[iam_role.ref],
instance_profile_name="{}-ec2-instance-profile".format(PRJ)
)
# Security Group
sg = ec2.CfnSecurityGroup(
self, "sg",
group_description="sg for ec2 instance({})".format(PRJ),
vpc_id=app_nw_stack.vpc.ref,
security_group_ingress=[]
)
# Instance
ec2.CfnInstance(
self, "instance",
iam_instance_profile=ec2_instance_profile.ref,
image_id=AMI_ID,
instance_type="t3.micro",
security_group_ids=[sg.ref],
subnet_id=app_nw_stack.ec2_subnet.ref,
tags=[nametag("instance")]
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
vpc = ec2.Vpc.from_lookup(self,'vpc',
vpc_id='vpc-082a9f3f7200f4513'
)
k8s_admin = iam.Role(self, "k8sadmin",
assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'),
role_name='eks-master-role',
managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name='AdministratorAccess'
)
]
)
k8s_instance_profile = iam.CfnInstanceProfile(self, 'instanceprofile',
roles=[k8s_admin.role_name],
instance_profile_name='eks-master-role'
)
cluster = eks.Cluster(self, 'dev',
cluster_name='eks-cdk-demo',
version='1.15',
vpc=vpc,
vpc_subnets=[ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE)],
default_capacity=0,
kubectl_enabled=True,
#security_group=k8s_sg,
masters_role= k8s_admin
)
#cluster.aws_auth.add_user_mapping(adminuser, {groups['system:masters']})
ng = cluster.add_nodegroup('eks-ng',
nodegroup_name='eks-ng',
instance_type=ec2.InstanceType('t3.medium'),
disk_size=5,
min_size=1,
max_size=1,
desired_size=1,
subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE),
remote_access=eks.NodegroupRemoteAccess(ssh_key_name='k8s-nodes')
)
def __init__(self, scope: core.Construct, construct_id: str, vpc: ec2.Vpc,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
env_name = self.node.try_get_context("env")
eks_role = iam.Role(
self,
"eksadmin",
assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'),
role_name='eks-cluster-role',
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
managed_policy_name='AdministratorAccess')
])
eks_instance_profile = iam.CfnInstanceProfile(
self,
'instanceprofile',
roles=[eks_role.role_name],
instance_profile_name='eks-cluster-role')
cluster = eks.Cluster(
self,
'prod',
cluster_name='ie-prod-snow-common',
version=eks.KubernetesVersion.V1_19,
vpc=vpc,
vpc_subnets=[
ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE)
],
default_capacity=0,
masters_role=eks_role)
nodegroup = cluster.add_nodegroup_capacity(
'eks-nodegroup',
instance_types=[
ec2.InstanceType('t3.large'),
ec2.InstanceType('m5.large'),
ec2.InstanceType('c5.large')
],
disk_size=50,
min_size=2,
max_size=2,
desired_size=2,
subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE),
remote_access=eks.NodegroupRemoteAccess(
ssh_key_name='ie-prod-snow-common'),
capacity_type=eks.CapacityType.SPOT)
def __init__(self,
scope: core.Construct,
id: str,
*,
role_name_prefix: Optional[str] = None,
artifacts_bucket: Optional[s3.Bucket] = None,
artifacts_path: Optional[str] = None,
logs_bucket: Optional[s3.Bucket] = None,
logs_path: Optional[str] = None) -> None:
super().__init__(scope, id)
if role_name_prefix:
self._service_role = EMRRoles._create_service_role(
self,
'EMRServiceRole',
role_name='{}-ServiceRole'.format(role_name_prefix))
self._instance_role = EMRRoles._create_instance_role(
self,
'EMRInstanceRole',
role_name='{}-InstanceRole'.format(role_name_prefix))
self._autoscaling_role = EMRRoles._create_autoscaling_role(
self,
'EMRAutoScalingRole',
role_name='{}-AutoScalingRole'.format(role_name_prefix))
self._instance_profile = iam.CfnInstanceProfile(
self,
'{}_InstanceProfile'.format(id),
roles=[self._instance_role.role_name],
instance_profile_name=self._instance_role.role_name)
self._instance_profile_arn = self._instance_profile.attr_arn
if artifacts_bucket:
artifacts_bucket.grant_read(
self._instance_role,
os.path.join(artifacts_path, '*')
if artifacts_path else artifacts_path).assert_success()
if logs_bucket:
logs_bucket.grant_read_write(
self._service_role,
os.path.join(logs_path, '*')
if logs_path else logs_path).assert_success()
logs_bucket.grant_read_write(
self._instance_role,
os.path.join(logs_path, '*')
if logs_path else logs_path).assert_success()
def create_emr_instance_role(scope: core.Construct) -> iam.Role:
"""
create an IAM (service) role for all EC2 instances (the service principal ec2) in our cluster
the role is assumed by nodes that run insied of EMR/Hadoop and hence they are allowed to talk to other
AWS services
under the hood this does pretty much the same as if someone would click the default roles in AWS EMR-UI
https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-role-for-ec2.html
"""
instance_role = iam.Role(
scope,
id=f"EmrInstanceRole",
assumed_by=iam.ServicePrincipal(f"ec2.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonElasticMapReduceforEC2Role")
],
)
instance_profile = iam.CfnInstanceProfile(scope,
"EmrInstanceProfile",
roles=[instance_role.role_name])
return iam.Role.from_role_arn(scope, "EmrInstaceProfileRole",
instance_profile.attr_arn)
def __init__(self, app: core.Construct, stack_name: str, vpc: aws_ec2.Vpc,
security_group: aws_ec2.SecurityGroup):
super().__init__(scope=app, id=f"{stack_name}-batch")
batch_role = aws_iam.Role(
scope=self,
id=f"batch_role",
role_name=f"batch_role",
assumed_by=aws_iam.ServicePrincipal("batch.amazonaws.com"))
batch_role.add_managed_policy(
aws_iam.ManagedPolicy.from_managed_policy_arn(
scope=self,
id=f"AWSBatchServiceRole",
managed_policy_arn=
"arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole"))
batch_role.add_to_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["arn:aws:logs:*:*:*"],
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogStreams"
]))
# Role to attach EC2
instance_role = aws_iam.Role(
scope=self,
id=f"instance_role",
role_name=f"instance_role_for",
assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com"))
instance_role.add_managed_policy(
aws_iam.ManagedPolicy.from_managed_policy_arn(
scope=self,
id=f"AmazonEC2ContainerServiceforEC2Role",
managed_policy_arn=
"arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
))
# add policy to access S3
instance_role.add_to_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["*"],
actions=["s3:*"]))
# add policy to access CloudWatch Logs
instance_role.add_to_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["arn:aws:logs:*:*:*"],
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogStreams"
]))
# attach role to EC2
instance_profile = aws_iam.CfnInstanceProfile(
scope=self,
id=f"instance_profile",
instance_profile_name=f"instance_profile",
roles=[instance_role.role_name])
# ===== #
# batch #
# ===== #
batch_compute_resources = aws_batch.ComputeResources(
vpc=vpc,
maxv_cpus=4,
minv_cpus=0,
security_groups=[security_group],
instance_role=instance_profile.attr_arn,
type=aws_batch.ComputeResourceType.SPOT)
batch_compute_environment = aws_batch.ComputeEnvironment(
scope=self,
id="batch_compute_environment",
compute_environment_name="batch_compute_environment",
compute_resources=batch_compute_resources,
service_role=batch_role)
job_role = aws_iam.Role(
scope=self,
id=f"job_role",
role_name=f"job_role",
assumed_by=aws_iam.ServicePrincipal("ecs-tasks.amazonaws.com"))
job_role.add_managed_policy(
aws_iam.ManagedPolicy.from_managed_policy_arn(
scope=self,
id=f"AmazonECSTaskExecutionRolePolicy",
managed_policy_arn=
"arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
))
job_role.add_managed_policy(
aws_iam.ManagedPolicy.from_managed_policy_arn(
scope=self,
id=f"AmazonS3FullAccess",
managed_policy_arn="arn:aws:iam::aws:policy/AmazonS3FullAccess"
))
job_role.add_managed_policy(
aws_iam.ManagedPolicy.from_managed_policy_arn(
scope=self,
id=f"CloudWatchLogsFullAccess",
managed_policy_arn=
"arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"))
self.batch_job_queue = aws_batch.JobQueue(
scope=self,
id=f"job_queue",
job_queue_name=f"job_queue",
compute_environments=[
aws_batch.JobQueueComputeEnvironment(
compute_environment=batch_compute_environment, order=1)
],
priority=1)
# ECR repository
ecr_repository = aws_ecr_assets.DockerImageAsset(
scope=self,
id=f"ecr_image",
directory="./docker",
repository_name=f"repository")
# get image from ECR
container_image = aws_ecs.ContainerImage.from_ecr_repository(
repository=ecr_repository.repository)
# job define
# pass `S3_BUCKET` as environment argument.
self.batch_job_definition = aws_batch.JobDefinition(
scope=self,
id=f"job_definition",
job_definition_name=f"job_definition",
container=aws_batch.JobDefinitionContainer(
image=container_image,
environment={"S3_BUCKET": f"{S3_BUCKET}"},
job_role=job_role,
vcpus=1,
memory_limit_mib=1024))
def __init__(self, scope: core.Construct, id: str, config_dict,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
""" Get VPC details """
vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id=config_dict['vpc_id'])
""" Create Security Group for Batch Env """
batch_security_group = "datalake-batch-security-group"
createBatchSecurityGroup = ec2.SecurityGroup(
self,
"createBatchSecurityGroup",
vpc=vpc,
allow_all_outbound=True,
description=
"This security group will be used for AWS Batch Compute Env",
security_group_name=batch_security_group)
createBatchSecurityGroup.add_ingress_rule(
peer=ec2.Peer.ipv4("0.0.0.0/0"),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="ingress_rule",
from_port=22,
to_port=22))
createBatchSecurityGroup.add_egress_rule(
peer=ec2.Peer.ipv4("0.0.0.0/0"),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
string_representation="egress_rule",
from_port=-1,
to_port=-1))
core.CfnOutput(self,
"createBatchSecurityGroupId",
value=createBatchSecurityGroup.security_group_id)
""" Create IAM Role for ecsInstance """
createECSInstanceRole = iam.Role(
self,
"createECSInstanceRole",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
description=
"This instance role will be used by the ECS cluster instances",
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEC2FullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AWSBatchFullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"SecretsManagerReadWrite"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonAthenaFullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/"
"AmazonEC2ContainerServiceforEC2Role"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSBatchServiceRole")
],
role_name="datalake-ecsinstance-role")
createInstanceProfile = iam.CfnInstanceProfile(
self,
"createInstanceProfile",
roles=[createECSInstanceRole.role_name],
instance_profile_name="datalake-ecsinstance-role")
useECSInstanceProfile = createInstanceProfile.instance_profile_name
core.CfnOutput(self,
"createECSInstanceRoleName",
value=createECSInstanceRole.role_name)
""" Create Spot Fleet Role """
createSpotFleetRole = iam.Role(
self,
'createSpotFleetRole',
assumed_by=iam.ServicePrincipal("spotfleet.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2SpotFleetTaggingRole")
])
core.CfnOutput(self,
"createSpotFleetRoleName",
value=createSpotFleetRole.role_name)
useSpotFleetRole = createSpotFleetRole.without_policy_updates()
""" Create Batch Service Role """
createBatchServiceRole = iam.Role(
self,
'createBatchServiceRole',
assumed_by=iam.ServicePrincipal("batch.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSBatchServiceRole")
])
core.CfnOutput(self,
"createBatchServiceRoleName",
value=createBatchServiceRole.role_name)
useBatchServiceRole = createBatchServiceRole.without_policy_updates()
""" Create Compute Environment """
subnet_1 = ec2.Subnet.from_subnet_attributes(
self,
"subnet_1",
subnet_id=config_dict['SubnetIds'].split(",")[0],
availability_zone=config_dict['AvailabilityZones'].split(",")[0])
subnet_2 = ec2.Subnet.from_subnet_attributes(
self,
"subnet_2",
subnet_id=config_dict['SubnetIds'].split(",")[1],
availability_zone=config_dict['AvailabilityZones'].split(",")[1])
createBatchComputeEnv = batch.ComputeEnvironment(
self,
"createBatchComputeEnv",
compute_environment_name="datalake-compute-env",
service_role=useBatchServiceRole,
compute_resources=batch.ComputeResources(
vpc=vpc,
type=batch.ComputeResourceType.SPOT,
bid_percentage=60,
desiredv_cpus=0,
maxv_cpus=100,
minv_cpus=0,
security_groups=[createBatchSecurityGroup],
vpc_subnets=ec2.SubnetSelection(subnets=[subnet_1, subnet_2]),
instance_role=useECSInstanceProfile,
spot_fleet_role=useSpotFleetRole,
compute_resources_tags=core.Tag.add(
self, 'Name', 'Datalake Pipeline Instance')))
core.CfnOutput(self,
"createBatchComputeEnvName",
value=createBatchComputeEnv.compute_environment_name)
getIComputeEnvObject = batch.ComputeEnvironment.from_compute_environment_arn(
self,
"getComputeEnvAtrributes",
compute_environment_arn=createBatchComputeEnv.
compute_environment_arn)
""" Create Batch Job Queue """
createBatchJobQueue = batch.JobQueue(
self,
"createBatchJobQueue",
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=getIComputeEnvObject, order=1)
],
enabled=True,
job_queue_name="datalake-job-queue",
priority=1)
core.CfnOutput(self,
"createBatchJobQueueName",
value=createBatchJobQueue.job_queue_name)
""" Create ECR Repo for datalake images """
createECRRepo = ecr.Repository(
self,
"createECRRepo",
repository_name=config_dict['workflow_ecr_repo'])
core.CfnOutput(self,
"createECRRepoName",
value=createECRRepo.repository_name)
def __init__(self, scope: core.Stack, id: str, **kwargs):
super().__init__(scope, id, **kwargs)
# This resource alone will create a private/public subnet in each AZ as well as nat/internet gateway(s)
self.vpc = aws_ec2.Vpc(
self,
"BaseVPC",
cidr='10.0.0.0/24',
)
# Creating ECS Cluster in the VPC created above
self.ecs_cluster = aws_ecs.Cluster(self,
"ECSCluster",
vpc=self.vpc,
cluster_name="container-demo")
# Adding service discovery namespace to cluster
self.ecs_cluster.add_default_cloud_map_namespace(name="service", )
###### EC2 SPOT CAPACITY PROVIDER SECTION ######
## As of today, AWS CDK doesn't support Launch Templates on the AutoScaling construct, hence it
## doesn't support Mixed Instances Policy to combine instance types on Auto Scaling and adhere to Spot best practices
## In the meantime, CfnLaunchTemplate and CfnAutoScalingGroup resources are used to configure Spot capacity
## https://github.com/aws/aws-cdk/issues/6734
self.ecs_spot_instance_role = aws_iam.Role(
self,
"ECSSpotECSInstanceRole",
assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com"),
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2ContainerServiceforEC2Role"),
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2RoleforSSM")
])
self.ecs_spot_instance_profile = aws_iam.CfnInstanceProfile(
self,
"ECSSpotInstanceProfile",
roles=[self.ecs_spot_instance_role.role_name])
## This creates a Launch Template for the Auto Scaling group
self.lt = aws_ec2.CfnLaunchTemplate(
self,
"ECSEC2SpotCapacityLaunchTemplate",
launch_template_data={
"instanceType":
"m5.large",
"imageId":
aws_ssm.StringParameter.value_for_string_parameter(
self,
"/aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id"
),
"securityGroupIds": [
x.security_group_id
for x in self.ecs_cluster.connections.security_groups
],
"iamInstanceProfile": {
"arn": self.ecs_spot_instance_profile.attr_arn
},
#
## Here we configure the ECS agent to drain Spot Instances upon catching a Spot Interruption notice from instance metadata
"userData":
core.Fn.base64(
core.Fn.sub(
"#!/usr/bin/bash\n"
"echo ECS_CLUSTER=${cluster_name} >> /etc/ecs/ecs.config\n"
"sudo iptables --insert FORWARD 1 --in-interface docker+ --destination 169.254.169.254/32 --jump DROP\n"
"sudo service iptables save\n"
"echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true >> /etc/ecs/ecs.config\n"
"echo ECS_AWSVPC_BLOCK_IMDS=true >> /etc/ecs/ecs.config\n"
"cat /etc/ecs/ecs.config",
variables={
"cluster_name": self.ecs_cluster.cluster_name
}))
},
launch_template_name="ECSEC2SpotCapacityLaunchTemplate")
self.ecs_ec2_spot_mig_asg = aws_autoscaling.CfnAutoScalingGroup(
self,
"ECSEC2SpotCapacity",
min_size="0",
max_size="10",
vpc_zone_identifier=[
x.subnet_id for x in self.vpc.private_subnets
],
mixed_instances_policy={
"instancesDistribution": {
"onDemandAllocationStrategy": "prioritized",
"onDemandBaseCapacity": 0,
"onDemandPercentageAboveBaseCapacity": 0,
"spotAllocationStrategy": "capacity-optimized"
},
"launchTemplate": {
"launchTemplateSpecification": {
"launchTemplateId": self.lt.ref,
"version": self.lt.attr_default_version_number
},
"overrides": [{
"instanceType": "m5.large"
}, {
"instanceType": "m5d.large"
}, {
"instanceType": "m5a.large"
}, {
"instanceType": "m5ad.large"
}, {
"instanceType": "m5n.large"
}, {
"instanceType": "m5dn.large"
}, {
"instanceType": "m3.large"
}, {
"instanceType": "m4.large"
}, {
"instanceType": "t3.large"
}, {
"instanceType": "t2.large"
}]
}
})
#
core.Tag.add(self.ecs_ec2_spot_mig_asg, "Name",
self.ecs_ec2_spot_mig_asg.node.path)
core.CfnOutput(self,
"EC2SpotAutoScalingGroupName",
value=self.ecs_ec2_spot_mig_asg.ref,
export_name="EC2SpotASGName")
#
##### END EC2 SPOT CAPACITY PROVIDER SECTION #####
# Namespace details as CFN output
self.namespace_outputs = {
'ARN':
self.ecs_cluster.default_cloud_map_namespace.
private_dns_namespace_arn,
'NAME':
self.ecs_cluster.default_cloud_map_namespace.
private_dns_namespace_name,
'ID':
self.ecs_cluster.default_cloud_map_namespace.
private_dns_namespace_id,
}
# Cluster Attributes
self.cluster_outputs = {
'NAME': self.ecs_cluster.cluster_name,
'SECGRPS': str(self.ecs_cluster.connections.security_groups)
}
# When enabling EC2, we need the security groups "registered" to the cluster for imports in other service stacks
if self.ecs_cluster.connections.security_groups:
self.cluster_outputs['SECGRPS'] = str([
x.security_group_id
for x in self.ecs_cluster.connections.security_groups
][0])
# Frontend service to backend services on 3000
self.services_3000_sec_group = aws_ec2.SecurityGroup(
self,
"FrontendToBackendSecurityGroup",
allow_all_outbound=True,
description=
"Security group for frontend service to talk to backend services",
vpc=self.vpc)
# Allow inbound 3000 from ALB to Frontend Service
self.sec_grp_ingress_self_3000 = aws_ec2.CfnSecurityGroupIngress(
self,
"InboundSecGrp3000",
ip_protocol='TCP',
source_security_group_id=self.services_3000_sec_group.
security_group_id,
from_port=3000,
to_port=3000,
group_id=self.services_3000_sec_group.security_group_id)
# Creating an EC2 bastion host to perform load test on private backend services
amzn_linux = aws_ec2.MachineImage.latest_amazon_linux(
generation=aws_ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=aws_ec2.AmazonLinuxEdition.STANDARD,
virtualization=aws_ec2.AmazonLinuxVirt.HVM,
storage=aws_ec2.AmazonLinuxStorage.GENERAL_PURPOSE)
# Instance Role/profile that will be attached to the ec2 instance
# Enabling service role so the EC2 service can use ssm
role = aws_iam.Role(
self,
"InstanceSSM",
assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com"))
# Attaching the SSM policy to the role so we can use SSM to ssh into the ec2 instance
role.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2RoleforSSM"))
# Reading user data, to install siege into the ec2 instance.
with open("stresstool_user_data.sh") as f:
user_data = f.read()
# Instance creation
self.instance = aws_ec2.Instance(
self,
"Instance",
instance_name="{}-stresstool".format(stack_name),
instance_type=aws_ec2.InstanceType("t3.medium"),
machine_image=amzn_linux,
vpc=self.vpc,
role=role,
user_data=aws_ec2.UserData.custom(user_data),
security_group=self.services_3000_sec_group)
# All Outputs required for other stacks to build
core.CfnOutput(self,
"NSArn",
value=self.namespace_outputs['ARN'],
export_name="NSARN")
core.CfnOutput(self,
"NSName",
value=self.namespace_outputs['NAME'],
export_name="NSNAME")
core.CfnOutput(self,
"NSId",
value=self.namespace_outputs['ID'],
export_name="NSID")
core.CfnOutput(self,
"FE2BESecGrp",
value=self.services_3000_sec_group.security_group_id,
export_name="SecGrpId")
core.CfnOutput(self,
"ECSClusterName",
value=self.cluster_outputs['NAME'],
export_name="ECSClusterName")
core.CfnOutput(self,
"ECSClusterSecGrp",
value=self.cluster_outputs['SECGRPS'],
export_name="ECSSecGrpList")
core.CfnOutput(self,
"ServicesSecGrp",
value=self.services_3000_sec_group.security_group_id,
export_name="ServicesSecGrp")
core.CfnOutput(self,
"StressToolEc2Id",
value=self.instance.instance_id)
core.CfnOutput(self,
"StressToolEc2Ip",
value=self.instance.instance_private_ip)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
eks_vpc = ec2.Vpc(self, "VPC", cidr="10.0.0.0/16")
self.node.apply_aspect(
core.Tag("kubernetes.io/cluster/cluster", "shared"))
eks_vpc.private_subnets[0].node.apply_aspect(
core.Tag("kubernetes.io/role/internal-elb", "1"))
eks_vpc.private_subnets[1].node.apply_aspect(
core.Tag("kubernetes.io/role/internal-elb", "1"))
eks_vpc.public_subnets[0].node.apply_aspect(
core.Tag("kubernetes.io/role/elb", "1"))
eks_vpc.public_subnets[1].node.apply_aspect(
core.Tag("kubernetes.io/role/elb", "1"))
# Create IAM Role For CodeBuild and Cloud9
codebuild_role = iam.Role(
self,
"BuildRole",
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal("codebuild.amazonaws.com"),
iam.ServicePrincipal("ec2.amazonaws.com")),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AdministratorAccess")
])
instance_profile = iam.CfnInstanceProfile(
self, "InstanceProfile", roles=[codebuild_role.role_name])
# Create CodeBuild PipelineProject
build_project = codebuild.PipelineProject(
self,
"BuildProject",
role=codebuild_role,
build_spec=codebuild.BuildSpec.from_source_filename(
"buildspec.yml"))
# Create CodePipeline
pipeline = codepipeline.Pipeline(
self,
"Pipeline",
)
# Create Artifact
artifact = codepipeline.Artifact()
# S3 Source Bucket
source_bucket = s3.Bucket.from_bucket_attributes(
self,
"SourceBucket",
bucket_arn=core.Fn.join(
"",
["arn:aws:s3:::ee-assets-prod-",
core.Fn.ref("AWS::Region")]))
# Add Source Stage
pipeline.add_stage(
stage_name="Source",
actions=[
codepipeline_actions.S3SourceAction(
action_name="S3SourceRepo",
bucket=source_bucket,
bucket_key=
"modules/2cae1f20008d4fc5aaef294602649b98/v9/source.zip",
output=artifact,
trigger=codepipeline_actions.S3Trigger.NONE)
])
# Add CodeBuild Stage
pipeline.add_stage(
stage_name="Deploy",
actions=[
codepipeline_actions.CodeBuildAction(
action_name="CodeBuildProject",
project=build_project,
type=codepipeline_actions.CodeBuildActionType.BUILD,
input=artifact,
environment_variables={
'PublicSubnet1ID':
codebuild.BuildEnvironmentVariable(
value=eks_vpc.public_subnets[0].subnet_id),
'PublicSubnet2ID':
codebuild.BuildEnvironmentVariable(
value=eks_vpc.public_subnets[1].subnet_id),
'PrivateSubnet1ID':
codebuild.BuildEnvironmentVariable(
value=eks_vpc.private_subnets[0].subnet_id),
'PrivateSubnet2ID':
codebuild.BuildEnvironmentVariable(
value=eks_vpc.private_subnets[1].subnet_id),
'AWS_DEFAULT_REGION':
codebuild.BuildEnvironmentVariable(value=self.region),
'INSTANCEPROFILEID':
codebuild.BuildEnvironmentVariable(
value=instance_profile.ref),
'AWS_ACCOUNT_ID':
codebuild.BuildEnvironmentVariable(value=self.account)
})
])
cloud9_stack = cloudformation.CfnStack(
self,
"Cloud9Stack",
# template_url="https://aws-quickstart.s3.amazonaws.com/quickstart-cloud9-ide/templates/cloud9-ide-instance.yaml",
template_url=
"https://ee-assets-prod-us-east-1.s3.amazonaws.com/modules/2cae1f20008d4fc5aaef294602649b98/v9/cloud9-ide-instance.yaml",
parameters={
"C9InstanceType": "m5.large",
"C9Subnet": eks_vpc.public_subnets[0].subnet_id
})
pipeline.node.add_dependency(eks_vpc)
pipeline.node.add_dependency(cloud9_stack)
def __init__(self, scope: cdk.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
### Parameters
bootstrap_script_args = cdk.CfnParameter(self, 'BootstrapScriptArgs',
type='String',
default='',
description='Space seperated arguments passed to the bootstrap script.'
)
# create a VPC
vpc = ec2.Vpc(self, 'VPC', cidr='10.0.0.0/16', max_azs=99)
# create a private and public subnet per vpc
selection = vpc.select_subnets(
subnet_type=ec2.SubnetType.PRIVATE
)
# Output created subnets
for i, public_subnet in enumerate(vpc.public_subnets):
cdk.CfnOutput(self, 'PublicSubnet%i' % i, value=public_subnet.subnet_id)
for i, private_subnet in enumerate(vpc.private_subnets):
cdk.CfnOutput(self, 'PrivateSubnet%i' % i, value=private_subnet.subnet_id)
cdk.CfnOutput(self, 'VPCId', value=vpc.vpc_id)
# Create a Bucket
bucket = s3.Bucket(self, "DataRepository")
quickstart_bucket = s3.Bucket.from_bucket_name(self, 'QuickStartBucket', 'aws-quickstart')
# Upload Bootstrap Script to that bucket
bootstrap_script = assets.Asset(self, 'BootstrapScript',
path='scripts/bootstrap.sh'
)
# Upload parallel cluster post_install_script to that bucket
pcluster_post_install_script = assets.Asset(self, 'PclusterPostInstallScript',
path='scripts/post_install_script.sh'
)
# Setup CloudTrail
cloudtrail.Trail(self, 'CloudTrail', bucket=bucket)
# Create a Cloud9 instance
# Cloud9 doesn't have the ability to provide userdata
# Because of this we need to use SSM run command
cloud9_instance = cloud9.Ec2Environment(self, 'Cloud9Env', vpc=vpc, instance_type=ec2.InstanceType(instance_type_identifier='c5.large'))
cdk.CfnOutput(self, 'URL', value=cloud9_instance.ide_url)
# Create a keypair in lambda and store the private key in SecretsManager
c9_createkeypair_role = iam.Role(self, 'Cloud9CreateKeypairRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'))
c9_createkeypair_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole'))
# Add IAM permissions to the lambda role
c9_createkeypair_role.add_to_policy(iam.PolicyStatement(
actions=[
'ec2:CreateKeyPair',
'ec2:DeleteKeyPair'
],
resources=['*'],
))
# Lambda for Cloud9 keypair
c9_createkeypair_lambda = _lambda.Function(self, 'C9CreateKeyPairLambda',
runtime=_lambda.Runtime.PYTHON_3_6,
handler='lambda_function.handler',
timeout=cdk.Duration.seconds(300),
role=c9_createkeypair_role,
code=_lambda.Code.asset('functions/source/c9keypair'),
# code=_lambda.Code.from_bucket(
)
c9_createkeypair_provider = cr.Provider(self, "C9CreateKeyPairProvider", on_event_handler=c9_createkeypair_lambda)
c9_createkeypair_cr = cfn.CustomResource(self, "C9CreateKeyPair", provider=c9_createkeypair_provider,
properties={
'ServiceToken': c9_createkeypair_lambda.function_arn
}
)
#c9_createkeypair_cr.node.add_dependency(instance_id)
c9_ssh_private_key_secret = secretsmanager.CfnSecret(self, 'SshPrivateKeySecret',
secret_string=c9_createkeypair_cr.get_att_string('PrivateKey')
)
# The iam policy has a <REGION> parameter that needs to be replaced.
# We do it programmatically so future versions of the synth'd stack
# template include all regions.
with open('iam/ParallelClusterUserPolicy.json') as json_file:
data = json.load(json_file)
for s in data['Statement']:
if s['Sid'] == 'S3ParallelClusterReadOnly':
s['Resource'] = []
for r in region_info.RegionInfo.regions:
s['Resource'].append('arn:aws:s3:::{0}-aws-parallelcluster*'.format(r.name))
parallelcluster_user_policy = iam.CfnManagedPolicy(self, 'ParallelClusterUserPolicy', policy_document=iam.PolicyDocument.from_json(data))
# Cloud9 IAM Role
cloud9_role = iam.Role(self, 'Cloud9Role', assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'))
cloud9_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('AmazonSSMManagedInstanceCore'))
cloud9_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('AWSCloud9User'))
cloud9_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, 'AttachParallelClusterUserPolicy', parallelcluster_user_policy.ref))
cloud9_role.add_to_policy(iam.PolicyStatement(
resources=['*'],
actions=[
'ec2:DescribeInstances',
'ec2:DescribeVolumes',
'ec2:ModifyVolume'
]
))
cloud9_role.add_to_policy(iam.PolicyStatement(
resources=[c9_ssh_private_key_secret.ref],
actions=[
'secretsmanager:GetSecretValue'
]
))
bootstrap_script.grant_read(cloud9_role)
pcluster_post_install_script.grant_read(cloud9_role)
# Cloud9 User
# user = iam.User(self, 'Cloud9User', password=cdk.SecretValue.plain_text('supersecretpassword'), password_reset_required=True)
# Cloud9 Setup IAM Role
cloud9_setup_role = iam.Role(self, 'Cloud9SetupRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'))
cloud9_setup_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole'))
# Allow pcluster to be run in bootstrap
cloud9_setup_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, 'AttachParallelClusterUserPolicySetup', parallelcluster_user_policy.ref))
# Add IAM permissions to the lambda role
cloud9_setup_role.add_to_policy(iam.PolicyStatement(
actions=[
'cloudformation:DescribeStackResources',
'ec2:AssociateIamInstanceProfile',
'ec2:AuthorizeSecurityGroupIngress',
'ec2:DescribeInstances',
'ec2:DescribeInstanceStatus',
'ec2:DescribeInstanceAttribute',
'ec2:DescribeIamInstanceProfileAssociations',
'ec2:DescribeVolumes',
'ec2:DesctibeVolumeAttribute',
'ec2:DescribeVolumesModifications',
'ec2:DescribeVolumeStatus',
'ssm:DescribeInstanceInformation',
'ec2:ModifyVolume',
'ec2:ReplaceIamInstanceProfileAssociation',
'ec2:ReportInstanceStatus',
'ssm:SendCommand',
'ssm:GetCommandInvocation',
's3:GetObject',
'lambda:AddPermission',
'lambda:RemovePermission',
'events:PutRule',
'events:DeleteRule',
'events:PutTargets',
'events:RemoveTargets',
],
resources=['*'],
))
cloud9_setup_role.add_to_policy(iam.PolicyStatement(
actions=['iam:PassRole'],
resources=[cloud9_role.role_arn]
))
cloud9_setup_role.add_to_policy(iam.PolicyStatement(
actions=[
'lambda:AddPermission',
'lambda:RemovePermission'
],
resources=['*']
))
# Cloud9 Instance Profile
c9_instance_profile = iam.CfnInstanceProfile(self, "Cloud9InstanceProfile", roles=[cloud9_role.role_name])
# Lambda to add Instance Profile to Cloud9
c9_instance_profile_lambda = _lambda.Function(self, 'C9InstanceProfileLambda',
runtime=_lambda.Runtime.PYTHON_3_6,
handler='lambda_function.handler',
timeout=cdk.Duration.seconds(900),
role=cloud9_setup_role,
code=_lambda.Code.asset('functions/source/c9InstanceProfile'),
)
c9_instance_profile_provider = cr.Provider(self, "C9InstanceProfileProvider",
on_event_handler=c9_instance_profile_lambda,
)
instance_id = cfn.CustomResource(self, "C9InstanceProfile", provider=c9_instance_profile_provider,
properties={
'InstanceProfile': c9_instance_profile.ref,
'Cloud9Environment': cloud9_instance.environment_id,
}
)
instance_id.node.add_dependency(cloud9_instance)
# Lambda for Cloud9 Bootstrap
c9_bootstrap_lambda = _lambda.Function(self, 'C9BootstrapLambda',
runtime=_lambda.Runtime.PYTHON_3_6,
handler='lambda_function.handler',
timeout=cdk.Duration.seconds(900),
role=cloud9_setup_role,
code=_lambda.Code.asset('functions/source/c9bootstrap'),
)
c9_bootstrap_provider = cr.Provider(self, "C9BootstrapProvider", on_event_handler=c9_bootstrap_lambda)
c9_bootstrap_cr = cfn.CustomResource(self, "C9Bootstrap", provider=c9_bootstrap_provider,
properties={
'Cloud9Environment': cloud9_instance.environment_id,
'BootstrapPath': 's3://%s/%s' % (bootstrap_script.s3_bucket_name, bootstrap_script.s3_object_key),
'BootstrapArguments': bootstrap_script_args,
'VPCID': vpc.vpc_id,
'MasterSubnetID': vpc.public_subnets[0].subnet_id,
'ComputeSubnetID': vpc.private_subnets[0].subnet_id,
'PostInstallScriptS3Url': "".join( ['s3://', pcluster_post_install_script.s3_bucket_name, "/", pcluster_post_install_script.s3_object_key ] ),
'PostInstallScriptBucket': pcluster_post_install_script.s3_bucket_name,
'KeyPairId': c9_createkeypair_cr.ref,
'KeyPairSecretArn': c9_ssh_private_key_secret.ref
}
)
c9_bootstrap_cr.node.add_dependency(instance_id)
c9_bootstrap_cr.node.add_dependency(c9_createkeypair_cr)
c9_bootstrap_cr.node.add_dependency(c9_ssh_private_key_secret)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# get vpc used vpc-id
vpc = ec2.Vpc.from_lookup(self, "EKSNode", vpc_id=vpc_id)
subnets = []
if len(vpc.private_subnets) > 0:
subnets.extend(vpc.private_subnets)
elif len(vpc.public_subnets) > 0:
subnets.extend(vpc.public_subnets)
else:
print("Not any subnets found,")
return
# add a worker node role
workerRole = iam.Role.from_role_arn(self,
'nodeRole',
role_arn=nodes_role_arn)
#node_sg=ec2.SecurityGroup.from_security_group_id(self,"nodeSG",security_group_id='sg-0461d7bdfb942d0ef')
# add iam instance profile
instanceProfile = iam.CfnInstanceProfile(
self,
'kopsNodeProfile',
roles=[workerRole.role_name],
instance_profile_name='eks-cluster-workerNodeProfile')
# read and base64 encode userdata file
data = open('nodeup.sh', 'rb').read()
encodedBytes = base64.encodebytes(data)
encodedStr = str(encodedBytes, "utf-8")
# add worker instances and associate EIPs
for i in range(0, 1):
# add a network interface
eni0 = ec2.CfnNetworkInterface(
self,
'eni-' + str(i),
subnet_id='subnet-040455b57b16a4cc9',
group_set=['sg-0461d7bdfb942d0ef', 'sg-0b3ae225b27e04679'])
# add worker instances
instance = ec2.CfnInstance(
self,
"kops-node-" + str(i),
image_id=ami_id,
instance_type="t2.medium",
block_device_mappings=[
{
'deviceName': '/dev/xvda',
'ebs': {
'deleteOnTermination': True,
'volumeSize': 40,
'volumeType': 'gp2',
'encrypted': False,
},
},
{
'deviceName': '/dev/sdb', # for /var/lib/docker
'ebs': {
'deleteOnTermination': True,
'volumeSize': 100,
'volumeType': 'gp2',
'encrypted': False,
},
},
{
'deviceName': '/dev/sdc', # for data volume
'ebs': {
'deleteOnTermination': True,
'volumeSize': 200,
'volumeType': 'gp2',
'encrypted': False,
},
},
],
key_name=key_name,
network_interfaces=[{
'deviceIndex': '0',
'networkInterfaceId': eni0.ref,
}],
iam_instance_profile=instanceProfile.ref,
#iam_instance_profile=nodes_role_arn,
tags=[
core.CfnTag(key="KubernetesCluster",
value="eks-asgfleet-01"),
core.CfnTag(key="Name", value="test-01"),
core.CfnTag(key="k8s.io/role/node", value="1"),
core.CfnTag(key="CDK/manual", value="singlenode"),
],
user_data=encodedStr)
#associate EIP with the instance
eip = ec2.CfnEIP(self, "eip-" + str(i))
ec2.CfnEIPAssociation(self,
"eip-ass-i" + str(i),
allocation_id=eip.attr_allocation_id,
network_interface_id=eni0.ref)
def __init__(self, app: cdk.App, id: str, vpc: ec2.Vpc, servicedomain: str,
**kwargs) -> None:
super().__init__(app, id)
cluster = ecs.Cluster(self, id, vpc=vpc)
cluster.add_default_cloud_map_namespace(
name=servicedomain, type=ecs.NamespaceType.PrivateDns)
self._cluster = cluster
ecssg = ec2.SecurityGroup(self, 'ECSServiceSecurityGroup', vpc=vpc)
ecssg.add_ingress_rule(peer=ec2.CidrIPv4(vpc.vpc_cidr_block),
connection=ec2.TcpAllPorts())
self._clustersg = ecssg
# Bastion host stuff -------------------------------------------------------------------------------------
# BastionInstanceRole
pd = pu.PolicyUtils.createpolicyfromfile(
'./appmeshdemo/policydocs/appmesh.json')
bir = iam.Role(
self,
'BastionInstanceRole',
assumed_by=iam.ServicePrincipal('ec2'),
inline_policies={'appmesh': pd},
managed_policy_arns=[
'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM'
])
bip = iam.CfnInstanceProfile(self,
'BastionInstanceProfile',
roles=[bir.role_name])
# Bastion EC2 instance
bsg = ec2.SecurityGroup(self, 'BastionSG', vpc=vpc)
bsg.add_ingress_rule(peer=ec2.AnyIPv4(), connection=ec2.TcpAllPorts())
ni = ec2.CfnNetworkInterfaceProps()
ni['associatePublicIpAddress'] = True
ni['deviceIndex'] = '0'
ni['groupSet'] = [bsg.security_group_name]
ni['subnetId'] = vpc.public_subnets[0].subnet_id
bhi = ec2.CfnInstance(
self,
'BastionInstance',
instance_type='t2.micro',
iam_instance_profile=bip.instance_profile_name,
image_id=ec2.AmazonLinuxImage().get_image(self).image_id,
network_interfaces=[ni])
# Load-Balancer stuff ------------------------------------------------------------------------------------
plbsg = ec2.SecurityGroup(self, 'PublicLoadBalancerSG', vpc=vpc)
plbsg.add_ingress_rule(peer=ec2.AnyIPv4(),
connection=ec2.TcpPortRange(0, 65535))
plb = elbv2.ApplicationLoadBalancer(self,
'PublicLoadBalancer',
internet_facing=True,
load_balancer_name='appmeshdemo',
security_group=plbsg,
vpc=vpc,
idle_timeout_secs=30)
self._publoadbal = plb
healthchk = elbv2.HealthCheck()
healthchk['intervalSecs'] = 6
healthchk['healthyThresholdCount'] = 2
healthchk['unhealthyThresholdCount'] = 2
dtg = elbv2.ApplicationTargetGroup(
self,
'DummyTargetGroupPublic',
vpc=vpc,
port=80,
protocol=elbv2.ApplicationProtocol.Http,
health_check=healthchk,
target_group_name='appmeshdemo-drop-1')
plbl = elbv2.ApplicationListener(
self,
'PublicLoadBalancerListener',
load_balancer=plb,
port=80,
protocol=elbv2.ApplicationProtocol.Http,
default_target_groups=[dtg])
cdk.CfnOutput(self,
id='External URL',
value='http://' + plb.load_balancer_dns_name)
def __init__(
self,
scope: Construct,
construct_id: str,
*,
deploy_env: str,
processing_assets_table: aws_dynamodb.Table,
):
# pylint: disable=too-many-locals
super().__init__(scope, construct_id)
if deploy_env == "prod":
instance_types = [
aws_ec2.InstanceType("c5.xlarge"),
aws_ec2.InstanceType("c5.2xlarge"),
aws_ec2.InstanceType("c5.4xlarge"),
aws_ec2.InstanceType("c5.9xlarge"),
]
else:
instance_types = [
aws_ec2.InstanceType("m5.large"),
aws_ec2.InstanceType("m5.xlarge"),
]
ec2_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2ContainerServiceforEC2Role")
batch_instance_role = aws_iam.Role(
self,
"batch-instance-role",
assumed_by=aws_iam.ServicePrincipal(
"ec2.amazonaws.com"), # type: ignore[arg-type]
managed_policies=[ec2_policy],
)
processing_assets_table.grant_read_write_data(
batch_instance_role) # type: ignore[arg-type]
batch_instance_profile = aws_iam.CfnInstanceProfile(
self,
"batch-instance-profile",
roles=[batch_instance_role.role_name],
)
batch_launch_template_data = textwrap.dedent("""
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
--==MYBOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
echo ECS_IMAGE_PULL_BEHAVIOR=prefer-cached >> /etc/ecs/ecs.config
--==MYBOUNDARY==--
""")
launch_template_data = aws_ec2.CfnLaunchTemplate.LaunchTemplateDataProperty(
user_data=Fn.base64(batch_launch_template_data.strip()))
cloudformation_launch_template = aws_ec2.CfnLaunchTemplate(
self,
"batch-launch-template",
launch_template_name=f"{deploy_env}-datalake-batch-launch-template",
launch_template_data=launch_template_data,
)
assert cloudformation_launch_template.launch_template_name is not None
launch_template = aws_batch.LaunchTemplateSpecification(
launch_template_name=cloudformation_launch_template.
launch_template_name)
# use existing VPC in LINZ AWS account.
# VPC with these tags is required to exist in AWS account before being deployed.
# A VPC will not be deployed by this project.
vpc = aws_ec2.Vpc.from_lookup(
self,
"datalake-vpc",
tags={
APPLICATION_NAME_TAG_NAME: APPLICATION_NAME,
"ApplicationLayer": "networking",
},
)
compute_resources = aws_batch.ComputeResources(
vpc=vpc,
minv_cpus=0,
desiredv_cpus=0,
maxv_cpus=1000,
instance_types=instance_types,
instance_role=batch_instance_profile.instance_profile_name,
allocation_strategy=aws_batch.AllocationStrategy(
"BEST_FIT_PROGRESSIVE"),
launch_template=launch_template,
)
batch_service_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSBatchServiceRole")
service_role = aws_iam.Role(
self,
"batch-service-role",
assumed_by=aws_iam.ServicePrincipal(
"batch.amazonaws.com"), # type: ignore[arg-type]
managed_policies=[batch_service_policy],
)
compute_environment = aws_batch.ComputeEnvironment(
self,
"compute-environment",
compute_resources=compute_resources,
service_role=service_role, # type: ignore[arg-type]
)
self.job_queue = aws_batch.JobQueue(
scope,
f"{construct_id}-job-queue",
compute_environments=[
aws_batch.JobQueueComputeEnvironment(
compute_environment=compute_environment,
order=10 # type: ignore[arg-type]
),
],
priority=10,
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
if self.node.try_get_context('vpc_type'):
validate_cdk_json(self)
ES_LOADER_TIMEOUT = 600
######################################################################
# REGION mapping / ELB & Lambda Arch
######################################################################
elb_id_temp = region_info.FactName.ELBV2_ACCOUNT
elb_map_temp = region_info.RegionInfo.region_map(elb_id_temp)
region_dict = {}
for region in elb_map_temp:
# ELB account ID
region_dict[region] = {'ElbV2AccountId': elb_map_temp[region]}
# Lambda Arch
if region in ('us-east-1', 'us-east-2', 'us-west-2', 'ap-south-1',
'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1',
'eu-central-1', 'eu-west-1', 'eu-west-2'):
region_dict[region]['LambdaArch'] = (
aws_lambda.Architecture.ARM_64.name)
else:
region_dict[region]['LambdaArch'] = (
aws_lambda.Architecture.X86_64.name)
region_mapping = core.CfnMapping(
scope=self, id='RegionMap', mapping=region_dict)
######################################################################
# get params
######################################################################
allow_source_address = core.CfnParameter(
self, 'AllowedSourceIpAddresses', allowed_pattern=r'^[0-9./\s]*',
description='Space-delimited list of CIDR blocks',
default='10.0.0.0/8 172.16.0.0/12 192.168.0.0/16')
sns_email = core.CfnParameter(
self, 'SnsEmail', allowed_pattern=r'^[0-9a-zA-Z@_\-\+\.]*',
description=('Input your email as SNS topic, where Amazon '
'OpenSearch Service will send alerts to'),
default='*****@*****.**')
geoip_license_key = core.CfnParameter(
self, 'GeoLite2LicenseKey', allowed_pattern=r'^[0-9a-zA-Z]{16}$',
default='xxxxxxxxxxxxxxxx',
description=("If you wolud like to enrich geoip locaiton such as "
"IP address's country, get a license key form MaxMind"
" and input the key. If you not, keep "
"xxxxxxxxxxxxxxxx"))
reserved_concurrency = core.CfnParameter(
self, 'ReservedConcurrency', default=10, type='Number',
description=('Input reserved concurrency. Increase this value if '
'there are steady logs delay despite no errors'))
aes_domain_name = self.node.try_get_context('aes_domain_name')
bucket = f'{aes_domain_name}-{core.Aws.ACCOUNT_ID}'
s3bucket_name_geo = f'{bucket}-geo'
s3bucket_name_log = f'{bucket}-log'
s3bucket_name_snapshot = f'{bucket}-snapshot'
# organizations / multiaccount
org_id = self.node.try_get_context('organizations').get('org_id')
org_mgmt_id = self.node.try_get_context(
'organizations').get('management_id')
org_member_ids = self.node.try_get_context(
'organizations').get('member_ids')
no_org_ids = self.node.try_get_context(
'no_organizations').get('aws_accounts')
# Overwrite default S3 bucket name as customer name
temp_geo = self.node.try_get_context('s3_bucket_name').get('geo')
if temp_geo:
s3bucket_name_geo = temp_geo
else:
print('Using default bucket names')
temp_log = self.node.try_get_context('s3_bucket_name').get('log')
if temp_log:
s3bucket_name_log = temp_log
elif org_id or no_org_ids:
s3bucket_name_log = f'{aes_domain_name}-{self.account}-log'
else:
print('Using default bucket names')
temp_snap = self.node.try_get_context('s3_bucket_name').get('snapshot')
if temp_snap:
s3bucket_name_snapshot = temp_snap
else:
print('Using default bucket names')
kms_cmk_alias = self.node.try_get_context('kms_cmk_alias')
if not kms_cmk_alias:
kms_cmk_alias = 'aes-siem-key'
print('Using default key alais')
######################################################################
# deploy VPC when context is defined as using VPC
######################################################################
# vpc_type is 'new' or 'import' or None
vpc_type = self.node.try_get_context('vpc_type')
if vpc_type == 'new':
is_vpc = True
vpc_cidr = self.node.try_get_context('new_vpc_nw_cidr_block')
subnet_cidr_mask = int(
self.node.try_get_context('new_vpc_subnet_cidr_mask'))
is_vpc = True
# VPC
vpc_aes_siem = aws_ec2.Vpc(
self, 'VpcAesSiem', cidr=vpc_cidr,
max_azs=3, nat_gateways=0,
subnet_configuration=[
aws_ec2.SubnetConfiguration(
subnet_type=aws_ec2.SubnetType.ISOLATED,
name='aes-siem-subnet', cidr_mask=subnet_cidr_mask)])
subnet1 = vpc_aes_siem.isolated_subnets[0]
subnets = [{'subnet_type': aws_ec2.SubnetType.ISOLATED}]
vpc_subnets = aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.ISOLATED)
vpc_aes_siem_opt = vpc_aes_siem.node.default_child.cfn_options
vpc_aes_siem_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
for subnet in vpc_aes_siem.isolated_subnets:
subnet_opt = subnet.node.default_child.cfn_options
subnet_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
elif vpc_type == 'import':
vpc_id = self.node.try_get_context('imported_vpc_id')
vpc_aes_siem = aws_ec2.Vpc.from_lookup(
self, 'VpcAesSiem', vpc_id=vpc_id)
subnet_ids = get_subnet_ids(self)
subnets = []
for number, subnet_id in enumerate(subnet_ids, 1):
obj_id = 'Subenet' + str(number)
subnet = aws_ec2.Subnet.from_subnet_id(self, obj_id, subnet_id)
subnets.append(subnet)
subnet1 = subnets[0]
vpc_subnets = aws_ec2.SubnetSelection(subnets=subnets)
if vpc_type:
is_vpc = True
# Security Group
sg_vpc_noinbound_aes_siem = aws_ec2.SecurityGroup(
self, 'AesSiemVpcNoinboundSecurityGroup',
security_group_name='aes-siem-noinbound-vpc-sg',
vpc=vpc_aes_siem)
sg_vpc_aes_siem = aws_ec2.SecurityGroup(
self, 'AesSiemVpcSecurityGroup',
security_group_name='aes-siem-vpc-sg',
vpc=vpc_aes_siem)
sg_vpc_aes_siem.add_ingress_rule(
peer=aws_ec2.Peer.ipv4(vpc_aes_siem.vpc_cidr_block),
connection=aws_ec2.Port.tcp(443),)
sg_vpc_opt = sg_vpc_aes_siem.node.default_child.cfn_options
sg_vpc_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# VPC Endpoint
vpc_aes_siem.add_gateway_endpoint(
'S3Endpoint', service=aws_ec2.GatewayVpcEndpointAwsService.S3,
subnets=subnets)
vpc_aes_siem.add_interface_endpoint(
'SQSEndpoint', security_groups=[sg_vpc_aes_siem],
service=aws_ec2.InterfaceVpcEndpointAwsService.SQS,)
vpc_aes_siem.add_interface_endpoint(
'KMSEndpoint', security_groups=[sg_vpc_aes_siem],
service=aws_ec2.InterfaceVpcEndpointAwsService.KMS,)
else:
is_vpc = False
is_vpc = core.CfnCondition(
self, 'IsVpc', expression=core.Fn.condition_equals(is_vpc, True))
"""
CloudFormation実行時の条件式の書き方
ClassのBasesが aws_cdk.core.Resource の時は、
node.default_child.cfn_options.condition = is_vpc
ClassのBasesが aws_cdk.core.CfnResource の時は、
cfn_options.condition = is_vpc
"""
######################################################################
# create cmk of KMS to encrypt S3 bucket
######################################################################
kms_aes_siem = aws_kms.Key(
self, 'KmsAesSiemLog', description='CMK for SIEM solution',
removal_policy=core.RemovalPolicy.RETAIN)
aws_kms.Alias(
self, 'KmsAesSiemLogAlias', alias_name=kms_cmk_alias,
target_key=kms_aes_siem,
removal_policy=core.RemovalPolicy.RETAIN)
kms_aes_siem.add_to_resource_policy(
aws_iam.PolicyStatement(
sid='Allow GuardDuty to use the key',
actions=['kms:GenerateDataKey'],
principals=[aws_iam.ServicePrincipal(
'guardduty.amazonaws.com')],
resources=['*'],),)
kms_aes_siem.add_to_resource_policy(
aws_iam.PolicyStatement(
sid='Allow VPC Flow Logs to use the key',
actions=['kms:Encrypt', 'kms:Decrypt', 'kms:ReEncrypt*',
'kms:GenerateDataKey*', 'kms:DescribeKey'],
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
resources=['*'],),)
# basic policy
key_policy_basic1 = aws_iam.PolicyStatement(
sid='Allow principals in the account to decrypt log files',
actions=['kms:DescribeKey', 'kms:ReEncryptFrom'],
principals=[aws_iam.AccountPrincipal(
account_id=core.Aws.ACCOUNT_ID)],
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_basic1)
# for Athena
key_policy_athena = aws_iam.PolicyStatement(
sid='Allow Athena to query s3 objects with this key',
actions=['kms:Decrypt', 'kms:DescribeKey', 'kms:Encrypt',
'kms:GenerateDataKey*', 'kms:ReEncrypt*'],
principals=[aws_iam.AccountPrincipal(
account_id=core.Aws.ACCOUNT_ID)],
resources=['*'],
conditions={'ForAnyValue:StringEquals': {
'aws:CalledVia': 'athena.amazonaws.com'}})
kms_aes_siem.add_to_resource_policy(key_policy_athena)
# for CloudTrail
key_policy_trail1 = aws_iam.PolicyStatement(
sid='Allow CloudTrail to describe key',
actions=['kms:DescribeKey'],
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_trail1)
key_policy_trail2 = aws_iam.PolicyStatement(
sid=('Allow CloudTrail to encrypt logs'),
actions=['kms:GenerateDataKey*'],
principals=[aws_iam.ServicePrincipal(
'cloudtrail.amazonaws.com')],
resources=['*'],
conditions={'StringLike': {
'kms:EncryptionContext:aws:cloudtrail:arn': [
f'arn:aws:cloudtrail:*:{core.Aws.ACCOUNT_ID}:trail/*']}})
kms_aes_siem.add_to_resource_policy(key_policy_trail2)
######################################################################
# create s3 bucket
######################################################################
block_pub = aws_s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True
)
s3_geo = aws_s3.Bucket(
self, 'S3BucketForGeoip', block_public_access=block_pub,
bucket_name=s3bucket_name_geo,
# removal_policy=core.RemovalPolicy.DESTROY,
)
# create s3 bucket for log collector
s3_log = aws_s3.Bucket(
self, 'S3BucketForLog', block_public_access=block_pub,
bucket_name=s3bucket_name_log, versioned=True,
encryption=aws_s3.BucketEncryption.S3_MANAGED,
# removal_policy=core.RemovalPolicy.DESTROY,
)
# create s3 bucket for aes snapshot
s3_snapshot = aws_s3.Bucket(
self, 'S3BucketForSnapshot', block_public_access=block_pub,
bucket_name=s3bucket_name_snapshot,
# removal_policy=core.RemovalPolicy.DESTROY,
)
######################################################################
# IAM Role
######################################################################
# delopyment policy for lambda deploy-aes
arn_prefix = f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}'
loggroup_aes = f'log-group:/aws/aes/domains/{aes_domain_name}/*'
loggroup_opensearch = (
f'log-group:/aws/OpenSearchService/domains/{aes_domain_name}/*')
loggroup_lambda = 'log-group:/aws/lambda/aes-siem-*'
policydoc_create_loggroup = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=[
'logs:PutResourcePolicy',
'logs:DescribeLogGroups',
'logs:DescribeLogStreams'
],
resources=[f'{arn_prefix}:*', ]
),
aws_iam.PolicyStatement(
actions=[
'logs:CreateLogGroup', 'logs:CreateLogStream',
'logs:PutLogEvents', 'logs:PutRetentionPolicy'],
resources=[
f'{arn_prefix}:{loggroup_aes}',
f'{arn_prefix}:{loggroup_opensearch}',
f'{arn_prefix}:{loggroup_lambda}',
],
)
]
)
policydoc_crhelper = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=[
'lambda:AddPermission',
'lambda:RemovePermission',
'events:ListRules',
'events:PutRule',
'events:DeleteRule',
'events:PutTargets',
'events:RemoveTargets'],
resources=['*']
)
]
)
# snaphot rule for AES
policydoc_snapshot = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=['s3:ListBucket'],
resources=[s3_snapshot.bucket_arn]
),
aws_iam.PolicyStatement(
actions=['s3:GetObject', 's3:PutObject',
's3:DeleteObject'],
resources=[s3_snapshot.bucket_arn + '/*']
)
]
)
aes_siem_snapshot_role = aws_iam.Role(
self, 'AesSiemSnapshotRole',
role_name='aes-siem-snapshot-role',
inline_policies=[policydoc_snapshot, ],
assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com')
)
policydoc_assume_snapshrole = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=['iam:PassRole'],
resources=[aes_siem_snapshot_role.role_arn]
),
]
)
aes_siem_deploy_role_for_lambda = aws_iam.Role(
self, 'AesSiemDeployRoleForLambda',
role_name='aes-siem-deploy-role-for-lambda',
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonOpenSearchServiceFullAccess'),
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
],
inline_policies=[policydoc_assume_snapshrole, policydoc_snapshot,
policydoc_create_loggroup, policydoc_crhelper],
assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com')
)
if vpc_type:
aes_siem_deploy_role_for_lambda.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaVPCAccessExecutionRole')
)
# for alert from Amazon OpenSearch Service
aes_siem_sns_role = aws_iam.Role(
self, 'AesSiemSnsRole',
role_name='aes-siem-sns-role',
assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com')
)
# EC2 role
aes_siem_es_loader_ec2_role = aws_iam.Role(
self, 'AesSiemEsLoaderEC2Role',
role_name='aes-siem-es-loader-for-ec2',
assumed_by=aws_iam.ServicePrincipal('ec2.amazonaws.com'),
)
aws_iam.CfnInstanceProfile(
self, 'AesSiemEsLoaderEC2InstanceProfile',
instance_profile_name=aes_siem_es_loader_ec2_role.role_name,
roles=[aes_siem_es_loader_ec2_role.role_name]
)
######################################################################
# in VPC
######################################################################
aes_role_exist = check_iam_role('/aws-service-role/es.amazonaws.com/')
if vpc_type and not aes_role_exist:
slr_aes = aws_iam.CfnServiceLinkedRole(
self, 'AWSServiceRoleForAmazonOpenSearchService',
aws_service_name='es.amazonaws.com',
description='Created by cloudformation of siem stack'
)
slr_aes.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
######################################################################
# SQS for es-laoder's DLQ
######################################################################
sqs_aes_siem_dlq = aws_sqs.Queue(
self, 'AesSiemDlq', queue_name='aes-siem-dlq',
retention_period=core.Duration.days(14))
sqs_aes_siem_splitted_logs = aws_sqs.Queue(
self, 'AesSiemSqsSplitLogs',
queue_name='aes-siem-sqs-splitted-logs',
dead_letter_queue=aws_sqs.DeadLetterQueue(
max_receive_count=2, queue=sqs_aes_siem_dlq),
visibility_timeout=core.Duration.seconds(ES_LOADER_TIMEOUT),
retention_period=core.Duration.days(14))
######################################################################
# Setup Lambda
######################################################################
# setup lambda of es_loader
lambda_es_loader_vpc_kwargs = {}
if vpc_type:
lambda_es_loader_vpc_kwargs = {
'security_group': sg_vpc_noinbound_aes_siem,
'vpc': vpc_aes_siem,
'vpc_subnets': vpc_subnets,
}
lambda_es_loader = aws_lambda.Function(
self, 'LambdaEsLoader', **lambda_es_loader_vpc_kwargs,
function_name='aes-siem-es-loader',
description=f'{SOLUTION_NAME} / es-loader',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
# code=aws_lambda.Code.asset('../lambda/es_loader.zip'),
code=aws_lambda.Code.asset('../lambda/es_loader'),
handler='index.lambda_handler',
memory_size=2048,
timeout=core.Duration.seconds(ES_LOADER_TIMEOUT),
reserved_concurrent_executions=(
reserved_concurrency.value_as_number),
dead_letter_queue_enabled=True,
dead_letter_queue=sqs_aes_siem_dlq,
environment={
'GEOIP_BUCKET': s3bucket_name_geo, 'LOG_LEVEL': 'info',
'POWERTOOLS_LOGGER_LOG_EVENT': 'false',
'POWERTOOLS_SERVICE_NAME': 'es-loader',
'POWERTOOLS_METRICS_NAMESPACE': 'SIEM'})
es_loader_newver = lambda_es_loader.add_version(
name=__version__, description=__version__)
es_loader_opt = es_loader_newver.node.default_child.cfn_options
es_loader_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# send only
# sqs_aes_siem_dlq.grant(lambda_es_loader, 'sqs:SendMessage')
# send and reieve. but it must be loop
sqs_aes_siem_dlq.grant(
lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage',
'sqs:DeleteMessage', 'sqs:GetQueueAttributes')
sqs_aes_siem_splitted_logs.grant(
lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage',
'sqs:DeleteMessage', 'sqs:GetQueueAttributes')
lambda_es_loader.add_event_source(
aws_lambda_event_sources.SqsEventSource(
sqs_aes_siem_splitted_logs, batch_size=1))
# es-loaer on EC2 role
sqs_aes_siem_dlq.grant(
aes_siem_es_loader_ec2_role, 'sqs:GetQueue*', 'sqs:ListQueues*',
'sqs:ReceiveMessage*', 'sqs:DeleteMessage*')
lambda_geo = aws_lambda.Function(
self, 'LambdaGeoipDownloader',
function_name='aes-siem-geoip-downloader',
description=f'{SOLUTION_NAME} / geoip-downloader',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
code=aws_lambda.Code.asset('../lambda/geoip_downloader'),
handler='index.lambda_handler',
memory_size=320,
timeout=core.Duration.seconds(300),
environment={
's3bucket_name': s3bucket_name_geo,
'license_key': geoip_license_key.value_as_string,
}
)
lambda_geo_newver = lambda_geo.add_version(
name=__version__, description=__version__)
lamba_geo_opt = lambda_geo_newver.node.default_child.cfn_options
lamba_geo_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
######################################################################
# setup OpenSearch Service
######################################################################
lambda_deploy_es = aws_lambda.Function(
self, 'LambdaDeployAES',
function_name='aes-siem-deploy-aes',
description=f'{SOLUTION_NAME} / opensearch domain deployment',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
# code=aws_lambda.Code.asset('../lambda/deploy_es.zip'),
code=aws_lambda.Code.asset('../lambda/deploy_es'),
handler='index.aes_domain_handler',
memory_size=128,
timeout=core.Duration.seconds(300),
environment={
'accountid': core.Aws.ACCOUNT_ID,
'aes_domain_name': aes_domain_name,
'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn,
'es_loader_role': lambda_es_loader.role.role_arn,
'allow_source_address': allow_source_address.value_as_string,
},
role=aes_siem_deploy_role_for_lambda,
)
lambda_deploy_es.add_environment(
's3_snapshot', s3_snapshot.bucket_name)
if vpc_type:
lambda_deploy_es.add_environment(
'vpc_subnet_id', subnet1.subnet_id)
lambda_deploy_es.add_environment(
'security_group_id', sg_vpc_aes_siem.security_group_id)
else:
lambda_deploy_es.add_environment('vpc_subnet_id', 'None')
lambda_deploy_es.add_environment('security_group_id', 'None')
deploy_es_newver = lambda_deploy_es.add_version(
name=__version__, description=__version__)
deploy_es_opt = deploy_es_newver.node.default_child.cfn_options
deploy_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# execute lambda_deploy_es to deploy Amaozon ES Domain
aes_domain = aws_cloudformation.CfnCustomResource(
self, 'AesSiemDomainDeployedR2',
service_token=lambda_deploy_es.function_arn,)
aes_domain.add_override('Properties.ConfigVersion', __version__)
es_endpoint = aes_domain.get_att('es_endpoint').to_string()
lambda_es_loader.add_environment('ES_ENDPOINT', es_endpoint)
lambda_es_loader.add_environment(
'SQS_SPLITTED_LOGS_URL', sqs_aes_siem_splitted_logs.queue_url)
lambda_configure_es_vpc_kwargs = {}
if vpc_type:
lambda_configure_es_vpc_kwargs = {
'security_group': sg_vpc_noinbound_aes_siem,
'vpc': vpc_aes_siem,
'vpc_subnets': aws_ec2.SubnetSelection(subnets=[subnet1, ]), }
lambda_configure_es = aws_lambda.Function(
self, 'LambdaConfigureAES', **lambda_configure_es_vpc_kwargs,
function_name='aes-siem-configure-aes',
description=f'{SOLUTION_NAME} / opensearch configuration',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
code=aws_lambda.Code.asset('../lambda/deploy_es'),
handler='index.aes_config_handler',
memory_size=128,
timeout=core.Duration.seconds(300),
environment={
'accountid': core.Aws.ACCOUNT_ID,
'aes_domain_name': aes_domain_name,
'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn,
'es_loader_role': lambda_es_loader.role.role_arn,
'allow_source_address': allow_source_address.value_as_string,
'es_endpoint': es_endpoint,
},
role=aes_siem_deploy_role_for_lambda,
)
lambda_configure_es.add_environment(
's3_snapshot', s3_snapshot.bucket_name)
if vpc_type:
lambda_configure_es.add_environment(
'vpc_subnet_id', subnet1.subnet_id)
lambda_configure_es.add_environment(
'security_group_id', sg_vpc_aes_siem.security_group_id)
else:
lambda_configure_es.add_environment('vpc_subnet_id', 'None')
lambda_configure_es.add_environment('security_group_id', 'None')
configure_es_newver = lambda_configure_es.add_version(
name=__version__, description=__version__)
configure_es_opt = configure_es_newver.node.default_child.cfn_options
configure_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
aes_config = aws_cloudformation.CfnCustomResource(
self, 'AesSiemDomainConfiguredR2',
service_token=lambda_configure_es.function_arn,)
aes_config.add_override('Properties.ConfigVersion', __version__)
aes_config.add_depends_on(aes_domain)
aes_config.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
es_arn = (f'arn:aws:es:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}'
f':domain/{aes_domain_name}')
# grant permission to es_loader role
inline_policy_to_load_entries_into_es = aws_iam.Policy(
self, 'aes-siem-policy-to-load-entries-to-es',
policy_name='aes-siem-policy-to-load-entries-to-es',
statements=[
aws_iam.PolicyStatement(
actions=['es:*'],
resources=[es_arn + '/*', ]),
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_to_load_entries_into_es)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_to_load_entries_into_es)
# grant additional permission to es_loader role
additional_kms_cmks = self.node.try_get_context('additional_kms_cmks')
if additional_kms_cmks:
inline_policy_access_to_additional_cmks = aws_iam.Policy(
self, 'access_to_additional_cmks',
policy_name='access_to_additional_cmks',
statements=[
aws_iam.PolicyStatement(
actions=['kms:Decrypt'],
resources=sorted(set(additional_kms_cmks))
)
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_access_to_additional_cmks)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_access_to_additional_cmks)
additional_buckets = self.node.try_get_context('additional_s3_buckets')
if additional_buckets:
buckets_list = []
for bucket in additional_buckets:
buckets_list.append(f'arn:aws:s3:::{bucket}')
buckets_list.append(f'arn:aws:s3:::{bucket}/*')
inline_policy_access_to_additional_buckets = aws_iam.Policy(
self, 'access_to_additional_buckets',
policy_name='access_to_additional_buckets',
statements=[
aws_iam.PolicyStatement(
actions=['s3:GetObject*', 's3:GetBucket*', 's3:List*'],
resources=sorted(set(buckets_list))
)
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_access_to_additional_buckets)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_access_to_additional_buckets)
kms_aes_siem.grant_decrypt(lambda_es_loader)
kms_aes_siem.grant_decrypt(aes_siem_es_loader_ec2_role)
######################################################################
# s3 notification and grant permisssion
######################################################################
s3_geo.grant_read_write(lambda_geo)
s3_geo.grant_read(lambda_es_loader)
s3_geo.grant_read(aes_siem_es_loader_ec2_role)
s3_log.grant_read(lambda_es_loader)
s3_log.grant_read(aes_siem_es_loader_ec2_role)
# create s3 notification for es_loader
notification = aws_s3_notifications.LambdaDestination(lambda_es_loader)
# assign notification for the s3 PUT event type
# most log system use PUT, but also CLB use POST & Multipart Upload
s3_log.add_event_notification(
aws_s3.EventType.OBJECT_CREATED, notification,
aws_s3.NotificationKeyFilter(prefix='AWSLogs/'))
# For user logs, not AWS logs
s3_log.add_event_notification(
aws_s3.EventType.OBJECT_CREATED, notification,
aws_s3.NotificationKeyFilter(prefix='UserLogs/'))
# Download geoip to S3 once by executing lambda_geo
get_geodb = aws_cloudformation.CfnCustomResource(
self, 'ExecLambdaGeoipDownloader',
service_token=lambda_geo.function_arn,)
get_geodb.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
# Download geoip every day at 6PM UTC
rule = aws_events.Rule(
self, 'CwlRuleLambdaGeoipDownloaderDilly',
schedule=aws_events.Schedule.rate(core.Duration.hours(12)))
rule.add_target(aws_events_targets.LambdaFunction(lambda_geo))
######################################################################
# bucket policy
######################################################################
s3_awspath = s3_log.bucket_arn + '/AWSLogs/' + core.Aws.ACCOUNT_ID
bucket_policy_common1 = aws_iam.PolicyStatement(
sid='ELB Policy',
principals=[aws_iam.AccountPrincipal(
account_id=region_mapping.find_in_map(
core.Aws.REGION, 'ElbV2AccountId'))],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],)
# NLB / ALB / R53resolver / VPC Flow Logs
bucket_policy_elb1 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryAclCheck For ALB NLB R53Resolver Flowlogs',
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
actions=['s3:GetBucketAcl', 's3:ListBucket'],
resources=[s3_log.bucket_arn],)
bucket_policy_elb2 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryWrite For ALB NLB R53Resolver Flowlogs',
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_common1)
s3_log.add_to_resource_policy(bucket_policy_elb1)
s3_log.add_to_resource_policy(bucket_policy_elb2)
# CloudTrail
bucket_policy_trail1 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryAclCheck For Cloudtrail',
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:GetBucketAcl'], resources=[s3_log.bucket_arn],)
bucket_policy_trail2 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryWrite For CloudTrail',
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_trail1)
s3_log.add_to_resource_policy(bucket_policy_trail2)
# GuardDuty
bucket_policy_gd1 = aws_iam.PolicyStatement(
sid='Allow GuardDuty to use the getBucketLocation operation',
principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')],
actions=['s3:GetBucketLocation'], resources=[s3_log.bucket_arn],)
bucket_policy_gd2 = aws_iam.PolicyStatement(
sid='Allow GuardDuty to upload objects to the bucket',
principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_log.bucket_arn + '/*'],)
bucket_policy_gd5 = aws_iam.PolicyStatement(
sid='Deny non-HTTPS access', effect=aws_iam.Effect.DENY,
actions=['s3:*'], resources=[s3_log.bucket_arn + '/*'],
conditions={'Bool': {'aws:SecureTransport': 'false'}})
bucket_policy_gd5.add_any_principal()
s3_log.add_to_resource_policy(bucket_policy_gd1)
s3_log.add_to_resource_policy(bucket_policy_gd2)
s3_log.add_to_resource_policy(bucket_policy_gd5)
# Config
bucket_policy_config1 = aws_iam.PolicyStatement(
sid='AWSConfig BucketPermissionsCheck and BucketExistenceCheck',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:GetBucketAcl', 's3:ListBucket'],
resources=[s3_log.bucket_arn],)
bucket_policy_config2 = aws_iam.PolicyStatement(
sid='AWSConfigBucketDelivery',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/Config/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_config1)
s3_log.add_to_resource_policy(bucket_policy_config2)
# geoip
bucket_policy_geo1 = aws_iam.PolicyStatement(
sid='Allow geoip downloader and es-loader to read/write',
principals=[lambda_es_loader.role, lambda_geo.role],
actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'],
resources=[s3_geo.bucket_arn + '/*'],)
s3_geo.add_to_resource_policy(bucket_policy_geo1)
# ES Snapshot
bucket_policy_snapshot = aws_iam.PolicyStatement(
sid='Allow ES to store snapshot',
principals=[aes_siem_snapshot_role],
actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'],
resources=[s3_snapshot.bucket_arn + '/*'],)
s3_snapshot.add_to_resource_policy(bucket_policy_snapshot)
######################################################################
# for multiaccount / organizaitons
######################################################################
if org_id or no_org_ids:
##################################################################
# KMS key policy for multiaccount / organizaitons
##################################################################
# for CloudTrail
cond_tail2 = self.make_resource_list(
path='arn:aws:cloudtrail:*:', tail=':trail/*',
keys=self.list_without_none(org_mgmt_id, no_org_ids))
key_policy_mul_trail2 = aws_iam.PolicyStatement(
sid=('Allow CloudTrail to encrypt logs for multiaccounts'),
actions=['kms:GenerateDataKey*'],
principals=[aws_iam.ServicePrincipal(
'cloudtrail.amazonaws.com')],
resources=['*'],
conditions={'StringLike': {
'kms:EncryptionContext:aws:cloudtrail:arn': cond_tail2}})
kms_aes_siem.add_to_resource_policy(key_policy_mul_trail2)
# for replicaiton
key_policy_rep1 = aws_iam.PolicyStatement(
sid=('Enable cross account encrypt access for S3 Cross Region '
'Replication'),
actions=['kms:Encrypt'],
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_rep1)
##################################################################
# Buckdet Policy for multiaccount / organizaitons
##################################################################
s3_log_bucket_arn = 'arn:aws:s3:::' + s3bucket_name_log
# for CloudTrail
s3_mulpaths = self.make_resource_list(
path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/*',
keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids))
bucket_policy_org_trail = aws_iam.PolicyStatement(
sid='AWSCloudTrailWrite for Multiaccounts / Organizations',
principals=[
aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:PutObject'], resources=s3_mulpaths,
conditions={'StringEquals': {
's3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_org_trail)
# config
s3_conf_multpaths = self.make_resource_list(
path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/Config/*',
keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids))
bucket_policy_mul_config2 = aws_iam.PolicyStatement(
sid='AWSConfigBucketDelivery',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:PutObject'], resources=s3_conf_multpaths,
conditions={'StringEquals': {
's3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_mul_config2)
# for replication
bucket_policy_rep1 = aws_iam.PolicyStatement(
sid='PolicyForDestinationBucket / Permissions on objects',
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
actions=['s3:ReplicateDelete', 's3:ReplicateObject',
's3:ReplicateTags', 's3:GetObjectVersionTagging',
's3:ObjectOwnerOverrideToBucketOwner'],
resources=[f'{s3_log_bucket_arn}/*'])
bucket_policy_rep2 = aws_iam.PolicyStatement(
sid='PolicyForDestinationBucket / Permissions on bucket',
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
actions=['s3:List*', 's3:GetBucketVersioning',
's3:PutBucketVersioning'],
resources=[f'{s3_log_bucket_arn}'])
s3_log.add_to_resource_policy(bucket_policy_rep1)
s3_log.add_to_resource_policy(bucket_policy_rep2)
######################################################################
# SNS topic for Amazon OpenSearch Service Alert
######################################################################
sns_topic = aws_sns.Topic(
self, 'SnsTopic', topic_name='aes-siem-alert',
display_name='AES SIEM')
sns_topic.add_subscription(aws_sns_subscriptions.EmailSubscription(
email_address=sns_email.value_as_string))
sns_topic.grant_publish(aes_siem_sns_role)
######################################################################
# output of CFn
######################################################################
kibanaurl = f'https://{es_endpoint}/_dashboards/'
kibanaadmin = aes_domain.get_att('kibanaadmin').to_string()
kibanapass = aes_domain.get_att('kibanapass').to_string()
core.CfnOutput(self, 'RoleDeploy', export_name='role-deploy',
value=aes_siem_deploy_role_for_lambda.role_arn)
core.CfnOutput(self, 'DashboardsUrl', export_name='dashboards-url',
value=kibanaurl)
core.CfnOutput(self, 'DashboardsPassword',
export_name='dashboards-pass', value=kibanapass,
description=('Please change the password in OpenSearch '
'Dashboards ASAP'))
core.CfnOutput(self, 'DashboardsAdminID',
export_name='dashboards-admin', value=kibanaadmin)
def __init__(self, scope: core.Construct, id: str, custom: dict,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
region = self.region
#create S3 access role for ec2
ec2Role = iam.Role(
self,
"aws-cdk-handson-lab02-ec2role",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess")
])
instanceProfile = iam.CfnInstanceProfile(
self,
"aws-cdk-handson-lab02-ec2Profile",
roles=[ec2Role.role_name],
instance_profile_name="aws-cdk-handson-lab02-ec2Profile",
)
#create new VPC for lab02
#vpc = ec2.Vpc.from_lookup(self, "aws-cdk-handson-lab02-vpc",vpc_name="default") #使用默认的vpc
#创建新的vpc
vpc = ec2.Vpc(self,
id="aws-cdk-handson-lab02-vpc",
cidr="172.30.0.0/16",
nat_gateways=0,
subnet_configuration=[
{
"cidrMask": 24,
"name": "subnet-1-",
"subnetType": ec2.SubnetType.PUBLIC
},
{
"cidrMask": 24,
"name": "subnet-2-",
"subnetType": ec2.SubnetType.PUBLIC
},
{
"cidrMask": 24,
"name": "subnet-3-",
"subnetType": ec2.SubnetType.PUBLIC
},
])
#使用已有的安全组
#sg=ec2.SecurityGroup.from_security_group_id(self,"nodeSG",security_group_id='sg-0dd53aaa5c9eb8324')
#创建新的安全组
sg = ec2.CfnSecurityGroup(
self,
"aws-cdk-handson-lab02-ec2securitygroup",
group_description="this is aws-cdk-handson workshop",
group_name="aws-cdk-handson-lab02-ec2securitygroup",
security_group_ingress=[
{
"ipProtocol": "tcp",
"fromPort": 80,
"toPort": 80,
"cidrIp": "0.0.0.0/0",
},
{
"ipProtocol": "tcp",
"fromPort": 22,
"toPort": 22,
"cidrIp": "0.0.0.0/0",
},
{
"ipProtocol": "tcp",
"fromPort": 8080,
"toPort": 8080,
"cidrIp": "0.0.0.0/0",
},
],
vpc_id=vpc.vpc_id)
#read and base64 encode userdata file
data = open("../resource/httpd.sh", "rb").read()
encodedBytes = base64.encodebytes(data)
encodedStr = str(encodedBytes, "utf-8")
#create ec2 instances
ami = ec2.AmazonLinuxImage(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2)
#创建2台EC2
ec2Count = 2
for i in range(ec2Count):
eni0 = ec2.CfnNetworkInterface(
self,
"eni-" + str(i),
subnet_id=vpc.public_subnets[0].subnet_id,
group_set=[sg.attr_group_id])
#group_set=[sg.security_group_id]
instance = ec2.CfnInstance(
self,
"ec2-httpd-" + str(i),
image_id=ami.get_image(
self).image_id, #use Amazon Linux 2 AMI
instance_type="t3.micro",
key_name="wsu-cn-northwest-1", #这个是keypair的名字非常重要
tags=[
core.CfnTag(key="Name",
value="aws-cdk-lab02-ec2-" + str(i))
], #加上标签
iam_instance_profile=instanceProfile.ref,
user_data=encodedStr,
network_interfaces=[{
'deviceIndex': '0',
'networkInterfaceId': eni0.ref,
}])
core.CfnOutput(self,
"PublicIP-" + str(i),
export_name="PublicIP-" + str(i),
value=instance.attr_public_ip)
core.CfnOutput(self,
"PublicDNSName-" + str(i),
export_name="PublicDNSName-" + str(i),
value=instance.attr_public_dns_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
eks_vpc = ec2.Vpc(self, "VPC", cidr="10.0.0.0/16")
self.eks_vpc = eks_vpc
# Create IAM Role For code-server bastion
bastion_role = iam.Role(
self,
"BastionRole",
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal("ec2.amazonaws.com"),
iam.AccountRootPrincipal()),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AdministratorAccess")
])
self.bastion_role = bastion_role
# Create EC2 Instance Profile for that Role
instance_profile = iam.CfnInstanceProfile(
self, "InstanceProfile", roles=[bastion_role.role_name])
# Create SecurityGroup for the Control Plane ENIs
eks_security_group = ec2.SecurityGroup(self,
"EKSSecurityGroup",
vpc=eks_vpc,
allow_all_outbound=True)
eks_security_group.add_ingress_rule(ec2.Peer.ipv4('10.0.0.0/16'),
ec2.Port.all_traffic())
# Create an EKS Cluster
eks_cluster = eks.Cluster(
self,
"cluster",
cluster_name="cluster",
vpc=eks_vpc,
masters_role=bastion_role,
default_capacity_type=eks.DefaultCapacityType.NODEGROUP,
default_capacity_instance=ec2.InstanceType("m5.large"),
default_capacity=2,
security_group=eks_security_group,
endpoint_access=eks.EndpointAccess.PUBLIC_AND_PRIVATE,
version=eks.KubernetesVersion.V1_17)
self.cluster_cert = eks_cluster.cluster_certificate_authority_data
# Deploy ALB Ingress Controller
# Create the k8s Service account and corresponding IAM Role mapped via IRSA
alb_service_account = eks_cluster.add_service_account(
"alb-ingress-controller",
name="alb-ingress-controller",
namespace="kube-system")
# Create the PolicyStatements to attach to the role
# I couldn't find a way to get this to work with a PolicyDocument and there are 10 of these
alb_policy_statement_json_1 = {
"Effect":
"Allow",
"Action": [
"acm:DescribeCertificate", "acm:ListCertificates",
"acm:GetCertificate"
],
"Resource":
"*"
}
alb_policy_statement_json_2 = {
"Effect":
"Allow",
"Action": [
"ec2:AuthorizeSecurityGroupIngress", "ec2:CreateSecurityGroup",
"ec2:CreateTags", "ec2:DeleteTags", "ec2:DeleteSecurityGroup",
"ec2:DescribeAccountAttributes", "ec2:DescribeAddresses",
"ec2:DescribeInstances", "ec2:DescribeInstanceStatus",
"ec2:DescribeInternetGateways",
"ec2:DescribeNetworkInterfaces", "ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVpcs",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:RevokeSecurityGroupIngress"
],
"Resource":
"*"
}
alb_policy_statement_json_3 = {
"Effect":
"Allow",
"Action": [
"elasticloadbalancing:AddListenerCertificates",
"elasticloadbalancing:AddTags",
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:CreateRule",
"elasticloadbalancing:CreateTargetGroup",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:DeleteRule",
"elasticloadbalancing:DeleteTargetGroup",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:DescribeListenerCertificates",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"elasticloadbalancing:DescribeRules",
"elasticloadbalancing:DescribeSSLPolicies",
"elasticloadbalancing:DescribeTags",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetGroupAttributes",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:ModifyRule",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:ModifyTargetGroupAttributes",
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:RemoveListenerCertificates",
"elasticloadbalancing:RemoveTags",
"elasticloadbalancing:SetIpAddressType",
"elasticloadbalancing:SetSecurityGroups",
"elasticloadbalancing:SetSubnets",
"elasticloadbalancing:SetWebAcl"
],
"Resource":
"*"
}
alb_policy_statement_json_4 = {
"Effect":
"Allow",
"Action": [
"iam:CreateServiceLinkedRole", "iam:GetServerCertificate",
"iam:ListServerCertificates"
],
"Resource":
"*"
}
alb_policy_statement_json_5 = {
"Effect": "Allow",
"Action": ["cognito-idp:DescribeUserPoolClient"],
"Resource": "*"
}
alb_policy_statement_json_6 = {
"Effect":
"Allow",
"Action": [
"waf-regional:GetWebACLForResource", "waf-regional:GetWebACL",
"waf-regional:AssociateWebACL",
"waf-regional:DisassociateWebACL"
],
"Resource":
"*"
}
alb_policy_statement_json_7 = {
"Effect": "Allow",
"Action": ["tag:GetResources", "tag:TagResources"],
"Resource": "*"
}
alb_policy_statement_json_8 = {
"Effect": "Allow",
"Action": ["waf:GetWebACL"],
"Resource": "*"
}
alb_policy_statement_json_9 = {
"Effect":
"Allow",
"Action": [
"wafv2:GetWebACL", "wafv2:GetWebACLForResource",
"wafv2:AssociateWebACL", "wafv2:DisassociateWebACL"
],
"Resource":
"*"
}
alb_policy_statement_json_10 = {
"Effect":
"Allow",
"Action": [
"shield:DescribeProtection", "shield:GetSubscriptionState",
"shield:DeleteProtection", "shield:CreateProtection",
"shield:DescribeSubscription", "shield:ListProtections"
],
"Resource":
"*"
}
# Attach the necessary permissions
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_1))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_2))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_3))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_4))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_5))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_6))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_7))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_8))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_9))
alb_service_account.add_to_policy(
iam.PolicyStatement.from_json(alb_policy_statement_json_10))
# Deploy the ALB Ingress Controller from the Helm chart
eks_cluster.add_helm_chart(
"aws-alb-ingress-controller",
chart="aws-alb-ingress-controller",
repository=
"http://storage.googleapis.com/kubernetes-charts-incubator",
namespace="kube-system",
values={
"clusterName": "cluster",
"awsRegion": os.environ["CDK_DEFAULT_REGION"],
"awsVpcID": eks_vpc.vpc_id,
"rbac": {
"create": True,
"serviceAccount": {
"create": False,
"name": "alb-ingress-controller"
}
}
})
# Create code-server bastion
# Get Latest Amazon Linux AMI
amzn_linux = ec2.MachineImage.latest_amazon_linux(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=ec2.AmazonLinuxEdition.STANDARD,
virtualization=ec2.AmazonLinuxVirt.HVM,
storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE)
# Create SecurityGroup for code-server
security_group = ec2.SecurityGroup(self,
"SecurityGroup",
vpc=eks_vpc,
allow_all_outbound=True)
security_group.add_ingress_rule(ec2.Peer.any_ipv4(),
ec2.Port.tcp(8080))
# Create our EC2 instance running CodeServer
code_server_instance = ec2.Instance(
self,
"CodeServerInstance",
instance_type=ec2.InstanceType("t3.large"),
machine_image=amzn_linux,
role=bastion_role,
vpc=eks_vpc,
vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC),
security_group=security_group,
block_devices=[
ec2.BlockDevice(device_name="/dev/xvda",
volume=ec2.BlockDeviceVolume.ebs(20))
])
# Add UserData
code_server_instance.user_data.add_commands(
"mkdir -p ~/.local/lib ~/.local/bin ~/.config/code-server")
code_server_instance.user_data.add_commands(
"curl -fL https://github.com/cdr/code-server/releases/download/v3.5.0/code-server-3.5.0-linux-amd64.tar.gz | tar -C ~/.local/lib -xz"
)
code_server_instance.user_data.add_commands(
"mv ~/.local/lib/code-server-3.5.0-linux-amd64 ~/.local/lib/code-server-3.5.0"
)
code_server_instance.user_data.add_commands(
"ln -s ~/.local/lib/code-server-3.5.0/bin/code-server ~/.local/bin/code-server"
)
code_server_instance.user_data.add_commands(
"echo \"bind-addr: 0.0.0.0:8080\" > ~/.config/code-server/config.yaml"
)
code_server_instance.user_data.add_commands(
"echo \"auth: password\" >> ~/.config/code-server/config.yaml")
code_server_instance.user_data.add_commands(
"echo \"password: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)\" >> ~/.config/code-server/config.yaml"
)
code_server_instance.user_data.add_commands(
"echo \"cert: false\" >> ~/.config/code-server/config.yaml")
code_server_instance.user_data.add_commands(
"~/.local/bin/code-server &")
code_server_instance.user_data.add_commands(
"yum -y install jq gettext bash-completion moreutils")
code_server_instance.user_data.add_commands(
"sudo pip install --upgrade awscli && hash -r")
code_server_instance.user_data.add_commands(
"echo 'export ALB_INGRESS_VERSION=\"v1.1.8\"' >> ~/.bash_profile")
code_server_instance.user_data.add_commands(
"curl --silent --location -o /usr/local/bin/kubectl \"https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.9/2020-08-04/bin/linux/amd64/kubectl\""
)
code_server_instance.user_data.add_commands(
"chmod +x /usr/local/bin/kubectl")
code_server_instance.user_data.add_commands(
"curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash"
)
code_server_instance.user_data.add_commands(
"export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account)"
)
code_server_instance.user_data.add_commands(
"export AWS_REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r '.region')"
)
code_server_instance.user_data.add_commands(
"echo \"export ACCOUNT_ID=${ACCOUNT_ID}\" | tee -a ~/.bash_profile"
)
code_server_instance.user_data.add_commands(
"echo \"export AWS_REGION=${AWS_REGION}\" | tee -a ~/.bash_profile"
)
code_server_instance.user_data.add_commands(
"aws configure set default.region ${AWS_REGION}")
code_server_instance.user_data.add_commands(
"curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"
)
code_server_instance.user_data.add_commands("yum -y install nodejs")
code_server_instance.user_data.add_commands(
"amazon-linux-extras enable python3")
code_server_instance.user_data.add_commands(
"yum install -y python3 --disablerepo amzn2-core")
code_server_instance.user_data.add_commands("yum install -y git")
code_server_instance.user_data.add_commands(
"rm /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python && ln -s /usr/bin/pip3 /usr/bin/pip"
)
code_server_instance.user_data.add_commands("npm install -g aws-cdk")
code_server_instance.user_data.add_commands(
"echo 'export KUBECONFIG=~/.kube/config' >> ~/.bash_profile")
code_server_instance.user_data.add_commands(
"git clone https://github.com/jasonumiker/eks-school.git")
# Add ALB
lb = elbv2.ApplicationLoadBalancer(self,
"LB",
vpc=eks_vpc,
internet_facing=True)
listener = lb.add_listener("Listener", port=80)
listener.connections.allow_default_port_from_any_ipv4(
"Open to the Internet")
listener.connections.allow_to_any_ipv4(
port_range=ec2.Port(string_representation="TCP 8080",
protocol=ec2.Protocol.TCP,
from_port=8080,
to_port=8080))
listener.add_targets(
"Target",
port=8080,
targets=[
elbv2.InstanceTarget(
instance_id=code_server_instance.instance_id, port=8080)
])
def __init__(self, scope: core.Construct, id: str,
landing_zone: ILandingZone,
directory: DirectoryServicesConstruct, group_names: [List],
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.__landing_zone = landing_zone
# Configure the security groups
self.security_group = ec2.SecurityGroup(
self,
'SecurityGroup',
vpc=landing_zone.networking.vpc,
allow_all_outbound=True,
description='HadoopConstruct Security Group',
security_group_name='hadoop-mapreduce-group')
for port in services.keys():
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
from_port=port,
to_port=port,
string_representation=services[port]))
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.UDP,
from_port=0,
to_port=65535,
string_representation='Allow All UDP Traffic'))
self.security_group.add_ingress_rule(
peer=ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol.TCP,
from_port=0,
to_port=65535,
string_representation='Allow All TCP Traffic'))
# Setup roles...
self.jobFlowRole = iam.Role(
self,
'JobFlowRole',
assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceforEC2Role'),
])
profile_name = 'jobflowprofile@{}-{}'.format(
landing_zone.zone_name,
core.Stack.of(self).region)
job_flow_instance_profile = iam.CfnInstanceProfile(
self,
'JobFlowInstanceProfile',
instance_profile_name=profile_name,
roles=[self.jobFlowRole.role_name])
serviceRole = iam.Role(
self,
'ServiceRole',
assumed_by=iam.ServicePrincipal(
service='elasticmapreduce.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceRole')
])
self.database = g.Database(self,
'GlueStore',
database_name='demo-database')
self.bucket = s3.Bucket(self,
'LogBucket',
removal_policy=core.RemovalPolicy.DESTROY)
emr_fs = EmrfsConstruct(self,
'Emrfs',
landing_zone=landing_zone,
directory=directory,
group_names=group_names,
job_flow_role=self.jobFlowRole)
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticmapreduce-instancefleetconfig.html
self.cluster = emr.CfnCluster(
self,
'Hadoop',
name='HadoopCluster',
job_flow_role=profile_name, #'EMR_EC2_DefaultRole',
service_role=serviceRole.role_name,
log_uri='s3://' + self.bucket.bucket_name + '/logs',
release_label='emr-6.2.0',
applications=[
emr.CfnCluster.ApplicationProperty(name='Spark'),
emr.CfnCluster.ApplicationProperty(name='Presto'),
emr.CfnCluster.ApplicationProperty(name='Hue'),
emr.CfnCluster.ApplicationProperty(name='Hive'),
emr.CfnCluster.ApplicationProperty(name='JupyterHub'),
],
configurations=[
emr.CfnCluster.ConfigurationProperty(
classification='spark-hive-site',
configuration_properties={
'hive.metastore.client.factory.class':
'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory'
}),
emr.CfnCluster.ConfigurationProperty(
classification='hive-site',
configuration_properties={
'hive.metastore.client.factory.class':
'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory',
'aws.glue.partition.num.segments':
'10', #1 to 10; (default=5)
'hive.metastore.schema.verification': 'false',
})
],
security_configuration=emr_fs.security_configuration.ref,
# kerberos_attributes= emr.CfnCluster.KerberosAttributesProperty(
# kdc_admin_password=directory.password,
# realm= directory.mad.name.upper(),
# ad_domain_join_password=directory.password,
# ad_domain_join_user= directory.admin
# ),
managed_scaling_policy=emr.CfnCluster.ManagedScalingPolicyProperty(
compute_limits=emr.CfnCluster.ComputeLimitsProperty(
minimum_capacity_units=1,
maximum_capacity_units=25,
unit_type='InstanceFleetUnits')),
instances=emr.CfnCluster.JobFlowInstancesConfigProperty(
#hadoop_version='2.4.0',
termination_protected=False,
master_instance_fleet=emr.CfnCluster.
InstanceFleetConfigProperty(
target_spot_capacity=1,
instance_type_configs=[
emr.CfnCluster.InstanceTypeConfigProperty(
instance_type='m5.xlarge', )
]),
core_instance_fleet=emr.CfnCluster.InstanceFleetConfigProperty(
target_spot_capacity=1,
instance_type_configs=[
emr.CfnCluster.InstanceTypeConfigProperty(
instance_type='m5.xlarge',
ebs_configuration=emr.CfnCluster.
EbsConfigurationProperty(ebs_block_device_configs=[
emr.CfnCluster.EbsBlockDeviceConfigProperty(
volume_specification=emr.CfnCluster.
VolumeSpecificationProperty(
size_in_gb=50, volume_type='gp2'))
]))
]),
additional_master_security_groups=[
self.security_group.security_group_id
],
additional_slave_security_groups=[
self.security_group.security_group_id
],
ec2_subnet_ids=[
net.subnet_id for net in landing_zone.networking.vpc.
_select_subnet_objects(subnet_group_name='Hadoop')
],
))
self.cluster.add_depends_on(job_flow_instance_profile)
def __init__(self, scope: core.Construct, id: str, props,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
################################################################################
# Set up permissions
ro_buckets = set()
for bucket in props['ro_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(self,
bucket,
bucket_name=bucket)
ro_buckets.add(tmp_bucket)
rw_buckets = set()
for bucket in props['rw_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(self,
bucket,
bucket_name=bucket)
rw_buckets.add(tmp_bucket)
batch_service_role = iam.Role(
self,
'BatchServiceRole',
assumed_by=iam.ServicePrincipal('batch.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSBatchServiceRole')
])
spotfleet_role = iam.Role(
self,
'AmazonEC2SpotFleetRole',
assumed_by=iam.ServicePrincipal('spotfleet.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2SpotFleetTaggingRole')
])
# Create role for Batch instances
batch_instance_role = iam.Role(
self,
'BatchInstanceRole',
role_name='RnasumBatchInstanceRole',
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal('ec2.amazonaws.com'),
iam.ServicePrincipal('ecs.amazonaws.com')),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2RoleforSSM'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2ContainerServiceforEC2Role')
])
batch_instance_role.add_to_policy(
iam.PolicyStatement(actions=[
"ec2:Describe*", "ec2:AttachVolume", "ec2:CreateVolume",
"ec2:CreateTags", "ec2:ModifyInstanceAttribute"
],
resources=["*"]))
batch_instance_role.add_to_policy(
iam.PolicyStatement(actions=["ecs:ListClusters"], resources=["*"]))
for bucket in ro_buckets:
bucket.grant_read(batch_instance_role)
for bucket in rw_buckets:
# TODO: restirct write to paths with */rnasum/*
bucket.grant_read_write(batch_instance_role)
# Turn the instance role into a Instance Profile
batch_instance_profile = iam.CfnInstanceProfile(
self,
'BatchInstanceProfile',
instance_profile_name='RnasumBatchInstanceProfile',
roles=[batch_instance_role.role_name])
################################################################################
# Minimal networking
# TODO: import resource created with TF
vpc = props['vpc']
################################################################################
# Setup Batch compute resources
# Configure BlockDevice to expand instance disk space (if needed?)
block_device_mappings = [{
'deviceName': '/dev/xvdf',
'ebs': {
'deleteOnTermination': True,
'volumeSize': 1024,
'volumeType': 'gp2'
}
}]
launch_template = ec2.CfnLaunchTemplate(
self,
'RnasumBatchComputeLaunchTemplate',
launch_template_name='RnasumBatchComputeLaunchTemplate',
launch_template_data={
# 'userData': core.Fn.base64(user_data_script), FIXME may not need this for RNAsum case? see job_definition below
'blockDeviceMappings': block_device_mappings
})
launch_template_spec = batch.LaunchTemplateSpecification(
launch_template_name=launch_template.launch_template_name,
version='$Latest')
my_compute_res = batch.ComputeResources(
type=batch.ComputeResourceType.SPOT,
allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE,
desiredv_cpus=0,
maxv_cpus=80,
minv_cpus=0,
image=ec2.MachineImage.generic_linux(
ami_map={'ap-southeast-2': props['compute_env_ami']}),
launch_template=launch_template_spec,
spot_fleet_role=spotfleet_role,
instance_role=batch_instance_profile.instance_profile_name,
vpc=vpc,
#compute_resources_tags=core.Tag('Creator', 'Batch')
)
# XXX: How to add more than one tag above??
# core.Tag.add(my_compute_res, 'Foo', 'Bar')
my_compute_env = batch.ComputeEnvironment(
self,
'RnasumBatchComputeEnv',
compute_environment_name="RnasumBatchComputeEnv",
service_role=batch_service_role,
compute_resources=my_compute_res)
job_queue = batch.JobQueue(self,
'RnasumJobQueue',
job_queue_name='rnasum_job_queue',
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=my_compute_env,
order=1)
],
priority=10)
# it is equivalent of
# https://github.com/umccr/infrastructure/blob/master/terraform/stacks/wts_report/jobs/wts_report.json
default_container_props = {
'image':
props['container_image'],
'vcpus':
2,
'memory':
2048,
'command': ['/opt/container/WTS-report-wrapper.sh', 'Ref::vcpus'],
'volumes': [{
'host': {
'sourcePath': '/mnt'
},
'name': 'work'
}, {
'host': {
'sourcePath': '/opt/container'
},
'name': 'container'
}],
'mountPoints': [{
'containerPath': '/work',
'readOnly': False,
'sourceVolume': 'work'
}, {
'containerPath': '/opt/container',
'readOnly': True,
'sourceVolume': 'container'
}],
'readonlyRootFilesystem':
False,
'privileged':
True,
'ulimits': []
}
# and CDK equivalent of
# https://github.com/umccr/infrastructure/blob/master/terraform/stacks/wts_report/main.tf#L113
job_definition = batch.CfnJobDefinition(
self,
'RnasumJobDefinition',
job_definition_name='rnasum_job_dev',
type='container',
container_properties=default_container_props,
parameters={
'vcpus': 1,
})
################################################################################
# Set up job submission Lambda
lambda_role = iam.Role(
self,
'RnasumLambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSBatchFullAccess') # TODO: restrict!
])
for bucket in ro_buckets:
bucket.grant_read(lambda_role)
for bucket in rw_buckets:
bucket.grant_read(lambda_role)
# TODO: support dev/prod split, i.e. image being configurable on dev, but fixed on prod
# may need a default JobDefinition to be set up
# and CDK equivalent of
# https://github.com/umccr/infrastructure/blob/master/terraform/stacks/wts_report/main.tf#L159
lmbda.Function(self,
'RnasumLambda',
function_name='rnasum_batch_lambda',
handler='trigger_wts_report.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas/'),
environment={
'JOBNAME_PREFIX': "rnasum_",
'JOBQUEUE': job_queue.job_queue_name,
'JOBDEF': job_definition.job_definition_name,
'REFDATA_BUCKET': props['refdata_bucket'],
'DATA_BUCKET': props['data_bucket'],
'JOB_MEM': '32000',
'JOB_VCPUS': '8',
'REF_DATASET': 'PANCAN',
'GENOME_BUILD': '38',
},
role=lambda_role)
def __init__(self, scope: core.Construct, id: str, props,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
################################################################################
# Set up permissions
ro_buckets = set()
for bucket in props['ro_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(self,
bucket,
bucket_name=bucket)
ro_buckets.add(tmp_bucket)
rw_buckets = set()
for bucket in props['rw_buckets']:
tmp_bucket = s3.Bucket.from_bucket_name(self,
bucket,
bucket_name=bucket)
rw_buckets.add(tmp_bucket)
batch_service_role = iam.Role(
self,
'BatchServiceRole',
assumed_by=iam.ServicePrincipal('batch.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSBatchServiceRole')
])
spotfleet_role = iam.Role(
self,
'AmazonEC2SpotFleetRole',
assumed_by=iam.ServicePrincipal('spotfleet.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2SpotFleetTaggingRole')
])
# Create role for Batch instances
batch_instance_role = iam.Role(
self,
'BatchInstanceRole',
role_name='UmccriseBatchInstanceRole',
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal('ec2.amazonaws.com'),
iam.ServicePrincipal('ecs.amazonaws.com')),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2RoleforSSM'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonEC2ContainerServiceforEC2Role')
])
batch_instance_role.add_to_policy(
iam.PolicyStatement(actions=[
"ec2:Describe*", "ec2:AttachVolume", "ec2:CreateVolume",
"ec2:CreateTags", "ec2:ModifyInstanceAttribute"
],
resources=["*"]))
batch_instance_role.add_to_policy(
iam.PolicyStatement(actions=["ecs:ListClusters"], resources=["*"]))
for bucket in ro_buckets:
bucket.grant_read(batch_instance_role)
for bucket in rw_buckets:
# restirct write to paths with */umccrise/*
bucket.grant_read_write(batch_instance_role, '*/umccrised/*')
# Turn the instance role into a Instance Profile
batch_instance_profile = iam.CfnInstanceProfile(
self,
'BatchInstanceProfile',
instance_profile_name='UmccriseBatchInstanceProfile',
roles=[batch_instance_role.role_name])
################################################################################
# Minimal networking
# TODO: import resource created with TF
vpc = props['vpc']
################################################################################
# Setup Batch compute resources
# Configure BlockDevice to expand instance disk space (if needed?)
block_device_mappings = [{
'deviceName': '/dev/xvdf',
'ebs': {
'deleteOnTermination': True,
'volumeSize': 1024,
'volumeType': 'gp2'
}
}]
launch_template = ec2.CfnLaunchTemplate(
self,
'UmccriseBatchComputeLaunchTemplate',
launch_template_name='UmccriseBatchComputeLaunchTemplate',
launch_template_data={
'userData': core.Fn.base64(user_data_script),
'blockDeviceMappings': block_device_mappings
})
launch_template_spec = batch.LaunchTemplateSpecification(
launch_template_name=launch_template.launch_template_name,
version='$Latest')
my_compute_res = batch.ComputeResources(
type=batch.ComputeResourceType.SPOT,
allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE,
desiredv_cpus=0,
maxv_cpus=128,
minv_cpus=0,
image=ec2.MachineImage.generic_linux(
ami_map={'ap-southeast-2': props['compute_env_ami']}),
launch_template=launch_template_spec,
spot_fleet_role=spotfleet_role,
instance_role=batch_instance_profile.instance_profile_name,
vpc=vpc,
#compute_resources_tags=core.Tag('Creator', 'Batch')
)
# XXX: How to add more than one tag above??
# core.Tag.add(my_compute_res, 'Foo', 'Bar')
my_compute_env = batch.ComputeEnvironment(
self,
'UmccriseBatchComputeEnv',
compute_environment_name="cdk-umccrise-batch-compute-env",
service_role=batch_service_role,
compute_resources=my_compute_res)
job_queue = batch.JobQueue(self,
'UmccriseJobQueue',
job_queue_name='cdk-umccrise_job_queue',
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=my_compute_env,
order=1)
],
priority=10)
job_container = batch.JobDefinitionContainer(
image=ecs.ContainerImage.from_registry(
name=props['container_image']),
vcpus=2,
memory_limit_mib=2048,
command=["/opt/container/umccrise-wrapper.sh", "Ref::vcpus"],
mount_points=[
ecs.MountPoint(container_path='/work',
read_only=False,
source_volume='work'),
ecs.MountPoint(container_path='/opt/container',
read_only=True,
source_volume='container')
],
volumes=[
ecs.Volume(name='container',
host=ecs.Host(source_path='/opt/container')),
ecs.Volume(name='work', host=ecs.Host(source_path='/mnt'))
],
privileged=True)
job_definition = batch.JobDefinition(
self,
'UmccriseJobDefinition',
job_definition_name='cdk-umccrise-job-definition',
parameters={'vcpus': '1'},
container=job_container,
timeout=core.Duration.hours(5))
################################################################################
# Set up job submission Lambda
lambda_role = iam.Role(
self,
'UmccriseLambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSBatchFullAccess') # TODO: restrict!
])
for bucket in ro_buckets:
bucket.grant_read(lambda_role)
for bucket in rw_buckets:
bucket.grant_read(lambda_role)
# TODO: support dev/prod split, i.e. image being configurable on dev, but fixed on prod
# may need a default JobDefinition to be set up
lmbda.Function(self,
'UmccriseLambda',
function_name='umccrise_batch_lambda',
handler='umccrise.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas/umccrise'),
environment={
'JOBNAME_PREFIX': "UMCCRISE_",
'JOBQUEUE': job_queue.job_queue_name,
'REFDATA_BUCKET': props['refdata_bucket'],
'DATA_BUCKET': props['data_bucket'],
'UMCCRISE_MEM': '50000',
'UMCCRISE_VCPUS': '16'
},
role=lambda_role)
def __init__(self, scope: core.Construct, **kwargs) -> None:
self.deploy_env = active_environment
super().__init__(scope,
id=f"{self.deploy_env.value}-databricks-stack",
**kwargs)
cross_account_role = iam.Role(
self,
id=f"iam-{self.deploy_env.value}-databricks-cross-account-role",
assumed_by=iam.AccountPrincipal(account_id="874361926784433"),
description=f"Allows databricks access to account",
)
cross_account_policy = iam.Policy(
self,
id=f"iam-{self.deploy_env.value}-databricks-cross-account-policy",
policy_name=
f"iam-{self.deploy_env.value}-databricks-cross-account-policy",
statements=[
iam.PolicyStatement(
actions=[
"ec2:AssociateDhcpOptions",
"ec2:AssociateIamInstanceProfile",
"ec2:AssociateRouteTable",
"ec2:AttachInternetGateway",
"ec2:AttachVolume",
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateDhcpOptions",
"ec2:CreateInternetGateway",
"ec2:CreateKeyPair",
"ec2:CreatePlacementGroup",
"ec2:CreateRoute",
"ec2:CreateSecurityGroup",
"ec2:CreateSubnet",
"ec2:CreateTags",
"ec2:CreateVolume",
"ec2:CreateVpc",
"ec2:CreateVpcPeeringConnection",
"ec2:DeleteInternetGateway",
"ec2:DeleteKeyPair",
"ec2:DeletePlacementGroup",
"ec2:DeleteRoute",
"ec2:DeleteRouteTable",
"ec2:DeleteSecurityGroup",
"ec2:DeleteSubnet",
"ec2:DeleteTags",
"ec2:DeleteVolume",
"ec2:DeleteVpc",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeIamInstanceProfileAssociations",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribePlacementGroups",
"ec2:DescribePrefixLists",
"ec2:DescribeReservedInstancesOfferings",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVolumes",
"ec2:DescribeVpcs",
"ec2:DetachInternetGateway",
"ec2:DisassociateIamInstanceProfile",
"ec2:ModifyVpcAttribute",
"ec2:ReplaceIamInstanceProfileAssociation",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RevokeSecurityGroupIngress",
"ec2:RunInstances",
"ec2:TerminateInstances",
],
resources=["*"],
),
iam.PolicyStatement(
actions=[
"iam:CreateServiceLinkedRole", "iam:PutRolePolicy"
],
resources=[
"arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"
],
conditions={
"StringEquals": {
"iam:AWSServiceName": "spot.amazonaws.com"
}
},
),
iam.PolicyStatement(
actions=[
"s3:GetBucketNotification",
"s3:PutBucketNotification",
"sns:ListSubscriptionsByTopic",
"sns:GetTopicAttributes",
"sns:SetTopicAttributes",
"sns:CreateTopic",
"sns:TagResource",
"sns:Publish",
"sns:Subscribe",
"sqs:CreateQueue",
"sqs:DeleteMessage",
"sqs:DeleteMessageBatch",
"sqs:ReceiveMessage",
"sqs:SendMessage",
"sqs:GetQueueUrl",
"sqs:GetQueueAttributes",
"sqs:SetQueueAttributes",
"sqs:TagQueue",
"sqs:ChangeMessageVisibility",
"sqs:ChangeMessageVisibilityBatch",
],
resources=[
f"arn:aws:s3:::s3-juan-armond-{self.deploy_env.value}-data-lake-*",
f"arn:aws:sqs:{self.region}:{self.account}:databricks-auto-ingest-*",
f"arn:aws:sns:{self.region}:{self.account}:databricks-auto-ingest-*",
],
),
iam.PolicyStatement(
actions=[
"sqs:ListQueues", "sqs:ListQueueTags", "sns:ListTopics"
],
resources=["*"],
),
iam.PolicyStatement(
actions=[
"sns:Unsubscribe", "sns:DeleteTopic", "sqs:DeleteQueue"
],
resources=[
f"arn:aws:sqs:{self.region}:{self.account}:databricks-auto-ingest-*",
f"arn:aws:sns:{self.region}:{self.account}:databricks-auto-ingest-*",
],
),
],
)
cross_account_role.attach_inline_policy(cross_account_policy)
bucket = s3.Bucket(
self,
id=f"s3-{self.deploy_env.value}-juan-armond-databricks-bucket",
bucket_name=
f"s3-{self.deploy_env.value}-juan-armond-databricks-bucket",
)
bucket.add_to_resource_policy(
iam.PolicyStatement(
principals=[iam.AccountPrincipal(account_id="414351767826")],
effect=iam.Effect.ALLOW,
actions=[
"s3:GetObject",
"s3:GetObjectVersion",
"s3:PutObject",
"s3:DeleteObject",
"s3:ListBucket",
"s3:GetBucketLocation",
],
resources=[bucket.bucket_arn, bucket.bucket_arn + "/*"],
))
access_role = iam.Role(
self,
id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-role",
assumed_by=iam.ServicePrincipal("ec2"),
description=f"Allows databricks access to data lake",
)
access_policy = iam.Policy(
self,
id=
f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy",
policy_name=
f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy",
statements=[
iam.PolicyStatement(
actions=[
"s3:ListBucket",
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject",
"s3:PutObjectAcl",
],
resources=[
f"arn:aws:s3:::s3-juan-armond-{self.deploy_env.value}-data-lake-*",
f"arn:aws:s3:::s3-juan-armond-{self.deploy_env.value}-data-lake-*/*",
],
),
iam.PolicyStatement(
actions=[
"iam:CreateServiceLinkedRole", "iam:PutRolePolicy"
],
resources=[
"arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"
],
conditions={
"StringEquals": {
"iam:AWSServiceName": "spot.amazonaws.com"
}
},
),
iam.PolicyStatement(
actions=[
"glue:BatchCreatePartition",
"glue:BatchDeletePartition",
"glue:BatchGetPartition",
"glue:CreateDatabase",
"glue:CreateTable",
"glue:CreateUserDefinedFunction",
"glue:DeleteDatabase",
"glue:DeletePartition",
"glue:DeleteTable",
"glue:DeleteUserDefinedFunction",
"glue:GetDatabase",
"glue:GetDatabases",
"glue:GetPartition",
"glue:GetPartitions",
"glue:GetTable",
"glue:GetTables",
"glue:GetUserDefinedFunction",
"glue:GetUserDefinedFunctions",
"glue:UpdateDatabase",
"glue:UpdatePartition",
"glue:UpdateTable",
"glue:UpdateUserDefinedFunction",
],
resources=["*"],
),
],
)
access_role.attach_inline_policy(access_policy)
cross_account_policy_data_access = iam.Policy(
self,
id=
f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access",
policy_name=
f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access",
statements=[
iam.PolicyStatement(
actions=[
"iam:PassRole",
],
resources=[access_role.role_arn],
)
],
)
iam.CfnInstanceProfile(
self,
id=
f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile",
instance_profile_name=
f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile",
roles=[access_role.role_name],
)
cross_account_role.attach_inline_policy(
cross_account_policy_data_access)
def __init__(self, scope: core.Construct, **kwargs) -> None:
self.deploy_env = active_environment
super().__init__(scope, id=f"{self.deploy_env.value}-databricks-stack", **kwargs)
cross_account_role = iam.Role(
self,
id=f"iam-{self.deploy_env.value}-databricks-cross-account-role",
assumed_by=iam.AccountPrincipal(account_id="414351767826"),
external_ids=["88086fcd-a382-4d73-9ddf-7e12326a3584"],
description=f"Allows databricks access to account",
)
cross_account_policy = iam.Policy(
self,
id=f"iam-{self.deploy_env.value}-databricks-cross-account-policy",
policy_name=f"iam-{self.deploy_env.value}-databricks-cross-account-policy",
statements=[
iam.PolicyStatement(
actions=[
"ec2:AllocateAddress",
"ec2:AssociateDhcpOptions",
"ec2:AssociateIamInstanceProfile",
"ec2:AssociateRouteTable",
"ec2:AttachInternetGateway",
"ec2:AttachVolume",
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateDhcpOptions",
"ec2:CreateInternetGateway",
"ec2:CreateKeyPair",
"ec2:CreateNatGateway",
"ec2:CreatePlacementGroup",
"ec2:CreateRoute",
"ec2:CreateRouteTable",
"ec2:CreateSecurityGroup",
"ec2:CreateSubnet",
"ec2:CreateTags",
"ec2:CreateVolume",
"ec2:CreateVpc",
"ec2:CreateVpcEndpoint",
"ec2:DeleteDhcpOptions",
"ec2:DeleteInternetGateway",
"ec2:DeleteKeyPair",
"ec2:DeleteNatGateway",
"ec2:DeletePlacementGroup",
"ec2:DeleteRoute",
"ec2:DeleteRouteTable",
"ec2:DeleteSecurityGroup",
"ec2:DeleteSubnet",
"ec2:DeleteTags",
"ec2:DeleteVolume",
"ec2:DeleteVpc",
"ec2:DeleteVpcEndpoints",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeIamInstanceProfileAssociations",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeInternetGateways",
"ec2:DescribeNatGateways",
"ec2:DescribePlacementGroups",
"ec2:DescribePrefixLists",
"ec2:DescribeReservedInstancesOfferings",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVolumes",
"ec2:DescribeVpcs",
"ec2:DetachInternetGateway",
"ec2:DisassociateIamInstanceProfile",
"ec2:DisassociateRouteTable",
"ec2:ModifyVpcAttribute",
"ec2:ReleaseAddress",
"ec2:ReplaceIamInstanceProfileAssociation",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RevokeSecurityGroupIngress",
"ec2:RunInstances",
"ec2:TerminateInstances",
],
resources=["*"],
),
iam.PolicyStatement(
actions=["iam:CreateServiceLinkedRole", "iam:PutRolePolicy"],
resources=[
"arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"
],
conditions={
"StringLike": {"iam:AWSServiceName": "spot.amazonaws.com"}
},
),
],
)
cross_account_role.attach_inline_policy(cross_account_policy)
bucket = s3.Bucket(
self,
id=f"s3-{self.deploy_env.value}-belisquito-databricks-bucket-turma-5",
bucket_name=f"s3-{self.deploy_env.value}-belisquito-databricks-bucket-turma-5",
)
bucket.add_to_resource_policy(
iam.PolicyStatement(
principals=[iam.AccountPrincipal(account_id="414351767826")],
effect=iam.Effect.ALLOW,
actions=[
"s3:GetObject",
"s3:GetObjectVersion",
"s3:PutObject",
"s3:DeleteObject",
"s3:ListBucket",
"s3:GetBucketLocation",
],
resources=[bucket.bucket_arn, bucket.bucket_arn + "/*"],
)
)
access_role = iam.Role(
self,
id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-role",
assumed_by=iam.ServicePrincipal("ec2"),
description=f"Allows databricks access to data lake",
)
access_policy = iam.Policy(
self,
id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy",
policy_name=f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy",
statements=[
iam.PolicyStatement(
actions=[
"s3:ListBucket",
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject",
"s3:PutObjectAcl",
],
resources=[
f"arn:aws:s3:::s3-belisquito-turma-5-{self.deploy_env.value}-data-lake-*",
f"arn:aws:s3:::s3-belisquito-turma-5-{self.deploy_env.value}-data-lake-*/*",
],
),
iam.PolicyStatement(
actions=["iam:CreateServiceLinkedRole", "iam:PutRolePolicy"],
resources=[
"arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"
],
conditions={
"StringEquals": {"iam:AWSServiceName": "spot.amazonaws.com"}
},
),
iam.PolicyStatement(
actions=[
"glue:BatchCreatePartition",
"glue:BatchDeletePartition",
"glue:BatchGetPartition",
"glue:CreateDatabase",
"glue:CreateTable",
"glue:CreateUserDefinedFunction",
"glue:DeleteDatabase",
"glue:DeletePartition",
"glue:DeleteTable",
"glue:DeleteUserDefinedFunction",
"glue:GetDatabase",
"glue:GetDatabases",
"glue:GetPartition",
"glue:GetPartitions",
"glue:GetTable",
"glue:GetTables",
"glue:GetUserDefinedFunction",
"glue:GetUserDefinedFunctions",
"glue:UpdateDatabase",
"glue:UpdatePartition",
"glue:UpdateTable",
"glue:UpdateUserDefinedFunction",
],
resources=["*"],
),
],
)
access_role.attach_inline_policy(access_policy)
cross_account_policy_data_access = iam.Policy(
self,
id=f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access",
policy_name=f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access",
statements=[
iam.PolicyStatement(
actions=[
"iam:PassRole",
],
resources=[access_role.role_arn],
)
],
)
iam.CfnInstanceProfile(
self,
id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile",
instance_profile_name=f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile",
roles=[access_role.role_name],
)
cross_account_role.attach_inline_policy(cross_account_policy_data_access)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
eks_vpc = ec2.Vpc(
self, "VPC",
cidr="10.0.0.0/16"
)
# Create IAM Role For EC2 bastion instance to be able to manage the cluster
bastion_role = iam.Role(
self, "BastionRole",
assumed_by=iam.CompositePrincipal(
iam.ServicePrincipal("ec2.amazonaws.com"),
iam.AccountRootPrincipal()
)
)
self.bastion_role = bastion_role
# Create EC2 Instance Profile for that Role
instance_profile = iam.CfnInstanceProfile(
self, "InstanceProfile",
roles=[bastion_role.role_name]
)
# Create SecurityGroup for the Control Plane ENIs
eks_security_group = ec2.SecurityGroup(
self, "EKSSecurityGroup",
vpc=eks_vpc,
allow_all_outbound=True
)
eks_security_group.add_ingress_rule(
ec2.Peer.ipv4('10.0.0.0/16'),
ec2.Port.all_traffic()
)
# Create an EKS Cluster
eks_cluster = eks.Cluster(
self, "cluster",
vpc=eks_vpc,
masters_role=bastion_role,
default_capacity_type=eks.DefaultCapacityType.NODEGROUP,
default_capacity_instance=ec2.InstanceType("m5.large"),
default_capacity=2,
security_group=eks_security_group,
endpoint_access=eks.EndpointAccess.PUBLIC_AND_PRIVATE,
version=eks.KubernetesVersion.V1_18
)
# Deploy ALB Ingress Controller
# Create the k8s Service account and corresponding IAM Role mapped via IRSA
alb_service_account = eks_cluster.add_service_account(
"aws-load-balancer-controller",
name="aws-load-balancer-controller",
namespace="kube-system"
)
# Create the PolicyStatements to attach to the role
# I couldn't find a way to get this to work with a PolicyDocument and there are 10 of these
alb_policy_statement_json_1 = {
"Effect": "Allow",
"Action": [
"acm:DescribeCertificate",
"acm:ListCertificates",
"acm:GetCertificate"
],
"Resource": "*"
}
alb_policy_statement_json_2 = {
"Effect": "Allow",
"Action": [
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteTags",
"ec2:DeleteSecurityGroup",
"ec2:DescribeAccountAttributes",
"ec2:DescribeAddresses",
"ec2:DescribeInstances",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInternetGateways",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSubnets",
"ec2:DescribeTags",
"ec2:DescribeVpcs",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:RevokeSecurityGroupIngress"
],
"Resource": "*"
}
alb_policy_statement_json_3 = {
"Effect": "Allow",
"Action": [
"elasticloadbalancing:AddListenerCertificates",
"elasticloadbalancing:AddTags",
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:CreateRule",
"elasticloadbalancing:CreateTargetGroup",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:DeleteRule",
"elasticloadbalancing:DeleteTargetGroup",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:DescribeListenerCertificates",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"elasticloadbalancing:DescribeRules",
"elasticloadbalancing:DescribeSSLPolicies",
"elasticloadbalancing:DescribeTags",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetGroupAttributes",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:ModifyRule",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:ModifyTargetGroupAttributes",
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:RemoveListenerCertificates",
"elasticloadbalancing:RemoveTags",
"elasticloadbalancing:SetIpAddressType",
"elasticloadbalancing:SetSecurityGroups",
"elasticloadbalancing:SetSubnets",
"elasticloadbalancing:SetWebAcl"
],
"Resource": "*"
}
alb_policy_statement_json_4 = {
"Effect": "Allow",
"Action": [
"iam:CreateServiceLinkedRole",
"iam:GetServerCertificate",
"iam:ListServerCertificates"
],
"Resource": "*"
}
alb_policy_statement_json_5 = {
"Effect": "Allow",
"Action": [
"cognito-idp:DescribeUserPoolClient"
],
"Resource": "*"
}
alb_policy_statement_json_6 = {
"Effect": "Allow",
"Action": [
"waf-regional:GetWebACLForResource",
"waf-regional:GetWebACL",
"waf-regional:AssociateWebACL",
"waf-regional:DisassociateWebACL"
],
"Resource": "*"
}
alb_policy_statement_json_7 = {
"Effect": "Allow",
"Action": [
"tag:GetResources",
"tag:TagResources"
],
"Resource": "*"
}
alb_policy_statement_json_8 = {
"Effect": "Allow",
"Action": [
"waf:GetWebACL"
],
"Resource": "*"
}
alb_policy_statement_json_9 = {
"Effect": "Allow",
"Action": [
"wafv2:GetWebACL",
"wafv2:GetWebACLForResource",
"wafv2:AssociateWebACL",
"wafv2:DisassociateWebACL"
],
"Resource": "*"
}
alb_policy_statement_json_10 = {
"Effect": "Allow",
"Action": [
"shield:DescribeProtection",
"shield:GetSubscriptionState",
"shield:DeleteProtection",
"shield:CreateProtection",
"shield:DescribeSubscription",
"shield:ListProtections"
],
"Resource": "*"
}
# Attach the necessary permissions
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_1))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_2))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_3))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_4))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_5))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_6))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_7))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_8))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_9))
alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_10))
# Deploy the ALB Ingress Controller from the Helm chart
eks_cluster.add_helm_chart(
"aws-load-balancer-controller",
chart="aws-load-balancer-controller",
repository="https://aws.github.io/eks-charts",
namespace="kube-system",
values={
"clusterName": eks_cluster.cluster_name,
"region": self.region,
"vpcId": eks_vpc.vpc_id,
"serviceAccount": {
"create": False,
"name": "aws-load-balancer-controller"
}
}
)
# Deploy External DNS Controller
# Create the k8s Service account and corresponding IAM Role mapped via IRSA
externaldns_service_account = eks_cluster.add_service_account(
"external-dns",
name="external-dns",
namespace="kube-system"
)
# Create the PolicyStatements to attach to the role
externaldns_policy_statement_json_1 = {
"Effect": "Allow",
"Action": [
"route53:ChangeResourceRecordSets"
],
"Resource": [
"arn:aws:route53:::hostedzone/*"
]
}
externaldns_policy_statement_json_2 = {
"Effect": "Allow",
"Action": [
"route53:ListHostedZones",
"route53:ListResourceRecordSets"
],
"Resource": [
"*"
]
}
# Add the policies to the service account
externaldns_service_account.add_to_policy(iam.PolicyStatement.from_json(externaldns_policy_statement_json_1))
externaldns_service_account.add_to_policy(iam.PolicyStatement.from_json(externaldns_policy_statement_json_2))
# Deploy the Helm Chart
eks_cluster.add_helm_chart(
"external-dns",
chart="external-dns",
repository="https://charts.bitnami.com/bitnami",
namespace="kube-system",
values={
"provider": "aws",
"aws": {
"region": self.region
},
"serviceAccount": {
"create": False,
"name": "external-dns"
},
"podSecurityContext": {
"fsGroup": 65534
}
}
)
# Install external secrets controller
# Create the Service Account
externalsecrets_service_account = eks_cluster.add_service_account(
"kubernetes-external-secrets",
name="kubernetes-external-secrets",
namespace="kube-system"
)
# Define the policy in JSON
externalsecrets_policy_statement_json_1 = {
"Effect": "Allow",
"Action": [
"secretsmanager:GetResourcePolicy",
"secretsmanager:GetSecretValue",
"secretsmanager:DescribeSecret",
"secretsmanager:ListSecretVersionIds"
],
"Resource": [
"*"
]
}
# Add the policies to the service account
externalsecrets_service_account.add_to_policy(iam.PolicyStatement.from_json(externalsecrets_policy_statement_json_1))
# Deploy the Helm Chart
eks_cluster.add_helm_chart(
"external-secrets",
chart="kubernetes-external-secrets",
repository="https://external-secrets.github.io/kubernetes-external-secrets/",
namespace="kube-system",
values={
"env": {
"AWS_REGION": self.region
},
"serviceAccount": {
"name": "kubernetes-external-secrets",
"create": False
},
"securityContext": {
"fsGroup": 65534
}
}
)
# Deploy Flux
# Deploy the Helm Chart
eks_cluster.add_helm_chart(
"flux",
chart="flux",
repository="https://charts.fluxcd.io",
namespace="kube-system",
values={
"git": {
"url": "[email protected]:jasonumiker/k8s-plus-aws-gitops",
"path": "k8s-app-resources",
"branch": "master"
}
}
)
# Deploy Prometheus and Grafana
# TODO Replace this with the new AWS Managed Prometheus and Grafana when available
eks_cluster.add_helm_chart(
"metrics",
chart="kube-prometheus-stack",
repository="https://prometheus-community.github.io/helm-charts",
namespace="monitoring",
values={
"prometheus": {
"prometheusSpec": {
"storageSpec": {
"volumeClaimTemplate": {
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "8Gi"
}
},
"storageClassName": "gp2"
}
}
}
}
},
"alertmanager": {
"alertmanagerSpec": {
"storage": {
"volumeClaimTemplate": {
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "2Gi"
}
},
"storageClassName": "gp2"
}
}
}
}
},
"grafana": {
"persistence": {
"enabled": "true",
"storageClassName": "gp2"
}
}
}
)
# Deploy Fluentbit and Elasticsearch
# Deploy an ElasticSearch Domain
es_domain = es.Domain(
self, "ESDomain",
version=es.ElasticsearchVersion.V7_9
)
# Create the Service Account
fluentbit_service_account = eks_cluster.add_service_account(
"fluentbit",
name="fluentbit",
namespace="monitoring"
)
# Define the policy in JSON
fluentbit_policy_statement_json_1 = {
"Effect": "Allow",
"Action": [
"es:ESHttp*"
],
"Resource": [
es_domain.domain_arn
]
}
# Add the policies to the service account
fluentbit_service_account.add_to_policy(iam.PolicyStatement.from_json(externalsecrets_policy_statement_json_1))
# Grant fluentbit access to our ES Domain
es_domain.grant_write(fluentbit_service_account)
eks_cluster.add_helm_chart(
"fluent-bit",
chart="fluent-bit",
repository="https://fluent.github.io/helm-charts",
namespace="monitoring",
values={
"serviceAccount": {
"create": False,
"name": "fluentbit"
},
"config": {
"outputs": "[OUTPUT]\n Name es\n Match *\n Host "+es_domain.domain_endpoint+"\n Port 443\n TLS On\n AWS_Auth On\n AWS_Region "+self.region+"\n Retry_Limit 6\n",
}
}
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# ========================
# VPC
# ========================
# VPC
vpc = ec2.Vpc(
self, 'fetch-and-run-vpc',
max_azs=2,
subnet_configuration=[
ec2.SubnetConfiguration(
name='public-subnet',
subnet_type=ec2.SubnetType.PUBLIC
)
],
nat_gateways=0
)
# Security Group
sg = ec2.SecurityGroup(
self, 'fetch-and-run-sg',
vpc=vpc,
description='SG for fetch and run',
security_group_name='fetch-and-run-sg'
)
# Ingress from IP address via HTTP, SSH
for port in PORTS:
sg.add_ingress_rule(
peer=ec2.Peer.ipv4(IP_ADDRESS),
connection=ec2.Port.tcp(port)
)
# ========================
# IAM
# ========================
'''
I. Batch Instance Role
- Makes calls to other AWS services on your behalf to
manage the resources that you use with the service
'''
batch_service_role = iam.Role.from_role_arn(
self, 'batch-service-role',
role_arn=BATCH_SERVICE_ROLE_ARN
)
'''
II. ECS Instance Role
- Batch compute environmens are populated with ECS container instances,
which run the ECS container agent locally
- ECS container agent makes calls to AWS APIs on your behalf
- Container instances that run the agent require a policy and role for
these services to know that the agent belongs to you
- Instance Profile uses the batch instance role name
- This is fed into the compute environment
'''
batch_instance_role = iam.Role.from_role_arn(
self, 'batch-instance-role',
role_arn=ECS_INSTANCE_ROLE_ARN
)
instance_profile = iam.CfnInstanceProfile(
self, 'instance-profile',
roles=[batch_instance_role.role_name]
)
'''
Job Role
- Used in the job definition
- IAM role that the container can assume for AWS permissions
When the fetch_and_run image runs as an AWS Batch job, it fetches the job
script from Amazon S3. You need an IAM role that the AWS Batch job can use
to access S3
Trusted Entity --> AWS service --> Elastic Container Service --> Elastic
Container Service Task
- In the Role's trust relationship, this will be displayed as follows:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
Default is for a role to be created
'''
batch_job_role = iam.Role.from_role_arn(
self, 'batch-job-role',
role_arn=BATCH_JOB_ROLE_ARN
)
# ========================
# ECR
# ========================
'''
Repository
TODO: Evaluate integrating repository into CDK (in this stack or another)
'''
ecr_repository = ecr.Repository.from_repository_name(
self, 'ecr-repository',
repository_name=ECR_REPOSITORY_NAME
)
'''
Container Image
NOTE: We are pulling the image directly from ECR. Pushed before stack is created.
- Can alternatively create the image from files in the stack (commented out)
TODO: Evaluate ability to programatically update the tag.
- Manually updating the tag follows approach of pushing image before stack creation/updates
- Review adding alphanumeric tag as opposed to simply 'latest' --> more detail for auditing
'''
# image_asset = ecr_assets.DockerImageAsset(
# self, 'docker-image',
# directory='./fetch-and-run',
# file='./Dockerfile'
# )
# image = ecs.ContainerImage.from_docker_image_asset(image_asset)
image = ecs.ContainerImage.from_ecr_repository(
repository=ecr_repository,
tag='latest'
)
# ========================
# BATCH
# ========================
'''
I. Compute Environment
- Execution runtime of submitted batch jobs
'''
compute_environment = batch.ComputeEnvironment(
self, 'batch-compute-environment',
compute_environment_name='batch-compute-environment',
compute_resources=batch.ComputeResources(
vpc=vpc,
# BEST_FIT_PROGRESSIVE will select an additional instance type that is large enough to meet the requirements of the jobs in the queue, with a preference for an instance type with a lower cost.
allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE,
compute_resources_tags={
"name": "fetch-and-run"
},
ec2_key_pair=KEY_PAIR,
instance_role=instance_profile.attr_arn,
security_groups=[sg],
type=batch.ComputeResourceType.ON_DEMAND,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PUBLIC)
),
service_role=batch_service_role,
)
'''
II. Job Queue
- Queue where batch jobs can be submitted
'''
job_queue = batch.JobQueue(
self, 'fetch-and-run-queue',
compute_environments=[
batch.JobQueueComputeEnvironment(
compute_environment=compute_environment,
order=1
)],
job_queue_name='fetch-and-run-queue'
)
'''
III. Job Definition
- Group various job properties (image, resource requirements, env variables)
into a single definition. Definitionns are used to job submission time
TODO: Build out functionality for the following:
- `command` => The command that is passed to the container. If you provide a shell command as a single string, you have to quote command-line arguments
- `environment` => The environment variables to pass to the container
- `mount_points` => The mount points for data volumes in your container
- `volumes` => A list of data volumes used in a job.
NOTE: Can optionally add command, environment variables directly in code
- Alternatively can reference them in `fetch_and_run.sh`
'''
job_definition = batch.JobDefinition(
self, 'fetch-and-run-job-definition',
container=batch.JobDefinitionContainer(
image=image,
job_role=batch_job_role,
# The hard limit (in MiB) of memory to present to the container
memory_limit_mib=500,
# The number of vCPUs reserved for the container. Each vCPU is equivalent to 1,024 CPU
vcpus=1,
user="******"
)
)
def __init__(self, scope: core.Construct, data_lake: DataLake,
common: Common, **kwargs) -> None:
self.env = data_lake.env.value
super().__init__(scope, id=f'{self.env}-emr-transform', **kwargs)
self.logs_bucket = s3.Bucket(
self,
f'{self.env}-emr-logs-bucket',
bucket_name=f's3-belisco-{self.env}-emr-logs-bucket',
removal_policy=core.RemovalPolicy.DESTROY)
buckets_arns = [
data_lake.data_lake_raw_bucket.bucket_arn,
data_lake.data_lake_processed_bucket.bucket_arn,
data_lake.data_lake_curated_bucket.bucket_arn
]
self.datalake_emr_policy = iam.Policy(
self,
id=f'iam-{self.env}-emr-data-lake',
policy_name=f'iam-{self.env}-emr-data-lake',
statements=[
iam.PolicyStatement(actions=[
's3:*',
],
resources=buckets_arns +
[f'{arn}/*' for arn in buckets_arns])
])
self.emr_role = iam.Role(
self,
f'{self.env}-emr-cluster-role',
assumed_by=iam.ServicePrincipal('elasticmapreduce.amazonaws.com'),
description='Role to allow EMR to process data',
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceRole')
])
self.emr_role.attach_inline_policy(self.datalake_emr_policy)
self.emr_ec2_role = iam.Role(
self,
f'{self.env}-emr-ec2-role',
assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'),
description='Role to allow EMR to process data',
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AmazonElasticMapReduceforEC2Role')
])
self.emr_ec2_role.attach_inline_policy(self.datalake_emr_policy)
self.emr_ec2_instance_profile = iam.CfnInstanceProfile(
self,
f'{self.env}-emr-instance_profile',
instance_profile_name=f'{self.env}-emr-instance_profile',
roles=[self.emr_ec2_role.role_name])
self.cluster = emr.CfnCluster(
self,
f'{self.env}-emr-cluster',
name=f'{self.env}-emr-cluster',
instances=emr.CfnCluster.JobFlowInstancesConfigProperty(
master_instance_group=emr.CfnCluster.
InstanceGroupConfigProperty(instance_count=1,
instance_type='m4.large',
market='ON_DEMAND',
name='Master'),
core_instance_group=emr.CfnCluster.InstanceGroupConfigProperty(
instance_count=2,
instance_type='m4.large',
market='ON_DEMAND',
name='Core'),
termination_protected=False,
ec2_subnet_id=common.custom_vpc.private_subnets[0].subnet_id),
applications=[emr.CfnCluster.ApplicationProperty(name='Spark')],
log_uri=f's3://{self.logs_bucket.bucket_name}/logs',
job_flow_role=self.emr_ec2_instance_profile.get_att(
'Arn').to_string(),
service_role=self.emr_role.role_arn,
release_label='emr-5.30.1',
visible_to_all_users=True,
configurations=[
emr.CfnCluster.ConfigurationProperty(
classification='spark-hive-site',
configuration_properties={
"hive.metastore.client.factory.class":
"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory"
})
])
