def create_instance_profile(self, job_role: iam.Role) -> iam.CfnInstanceProfile: return iam.CfnInstanceProfile( self, "emr_instance_profile", instance_profile_name=job_role.role_name, roles=[job_role.role_name], )
def add_instance_profile(self): iam.CfnInstanceProfile( self, id=f"iam-{self.deploy_env.value}-glue-data-lake-{self.layer.value}-instance-profile", instance_profile_name=f"iam-{self.deploy_env.value}-glue-data-lake-{self.layer.value}-instance-profile", roles=[self.role_name], )
def _add_instance_profile(self, cleanup_policy_statements, instance_role=None): """Set default instance profile in imagebuilder cfn template.""" instance_profile_resource = iam.CfnInstanceProfile( self, "InstanceProfile", path=IAM_ROLE_PATH, roles=[ instance_role.split("/")[-1] if instance_role else Fn.ref("InstanceRole") ], instance_profile_name=self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ) if not self.custom_cleanup_lambda_role: self._add_resource_delete_policy( cleanup_policy_statements, ["iam:DeleteInstanceProfile"], [ self.format_arn( service="iam", region="", resource="instance-profile", resource_name="{0}/{1}".format( IAM_ROLE_PATH.strip("/"), self._build_resource_name( IMAGEBUILDER_RESOURCE_NAME_PREFIX), ), ) ], ) return instance_profile_resource
def _add_ecs_instance_role_and_profile(self): ecs_instance_role = iam.CfnRole( self.stack_scope, "EcsInstanceRole", path=self._cluster_scoped_iam_path(), managed_policy_arns=[ self._format_arn( service="iam", account="aws", region="", resource= "policy/service-role/AmazonEC2ContainerServiceforEC2Role", ) ], assume_role_policy_document=get_assume_role_policy_document( f"ec2.{self._url_suffix}"), ) iam_instance_profile = iam.CfnInstanceProfile( self.stack_scope, "IamInstanceProfile", path=self._cluster_scoped_iam_path(), roles=[ecs_instance_role.ref]) return ecs_instance_role, iam_instance_profile
def add_instance_profile(self): instance_profile = iam.CfnInstanceProfile( self, id=f'iam-{self.environment}-glue-data-lake-instance-profile', instance_profile_name= f'iam-{self.environment}-glue-data-lake-instance-profile', roles=[self.role_name]) return instance_profile
def __init__( self, scope: core.Construct, common_stack: CommonResourcesStack, **kwargs, ) -> None: self.deploy_env = active_environment self.s3_script_bucket = common_stack.s3_script_bucket self.s3_log_bucket = common_stack.s3_log_bucket super().__init__(scope, id=f"{self.deploy_env.value}-emr-stack", **kwargs) # enable reading scripts from s3 bucket self.read_scripts_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "s3:GetObject", ], resources=[f"arn:aws:s3:::{self.s3_script_bucket.bucket_name}/*"], ) self.read_scripts_document = iam.PolicyDocument() self.read_scripts_document.add_statements(self.read_scripts_policy) # emr service role self.emr_service_role = iam.Role( self, "emr_service_role", assumed_by=iam.ServicePrincipal("elasticmapreduce.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonElasticMapReduceRole") ], inline_policies=[self.read_scripts_document], ) # emr job flow role self.emr_job_flow_role = iam.Role( self, "emr_job_flow_role", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonElasticMapReduceforEC2Role") ], ) # emr job flow profile self.emr_job_flow_profile = iam.CfnInstanceProfile( self, "emr_job_flow_profile", roles=[self.emr_job_flow_role.role_name], instance_profile_name="emrJobFlowProfile", )
def __init__(self, scope: core.Construct, id: str, app_nw_stack: appNwStack, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Contexts PRJ = self.node.try_get_context("prj") AMI_ID = self.node.try_get_context("ami-id") # Amazon Linux 2 # Functions def nametag(x): return core.CfnTag( key="Name", value="{}/{}".format(PRJ, x)) # ### Resources # IAM Role ec2_statement = iam.PolicyStatement() ec2_statement.add_actions("sts:AssumeRole") ec2_statement.add_service_principal(service="ec2.amazonaws.com") ec2_document = iam.PolicyDocument( statements=[ec2_statement] ) iam_role = iam.CfnRole( self, "iamRoleForEc2", role_name="{}-ec2-role".format(PRJ), description="{}-ec2-role".format(PRJ), assume_role_policy_document=ec2_document.to_json(), managed_policy_arns=[ "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM" ] ) ec2_instance_profile = iam.CfnInstanceProfile( self, "ec2InstanceProfile", roles=[iam_role.ref], instance_profile_name="{}-ec2-instance-profile".format(PRJ) ) # Security Group sg = ec2.CfnSecurityGroup( self, "sg", group_description="sg for ec2 instance({})".format(PRJ), vpc_id=app_nw_stack.vpc.ref, security_group_ingress=[] ) # Instance ec2.CfnInstance( self, "instance", iam_instance_profile=ec2_instance_profile.ref, image_id=AMI_ID, instance_type="t3.micro", security_group_ids=[sg.ref], subnet_id=app_nw_stack.ec2_subnet.ref, tags=[nametag("instance")] )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) vpc = ec2.Vpc.from_lookup(self,'vpc', vpc_id='vpc-082a9f3f7200f4513' ) k8s_admin = iam.Role(self, "k8sadmin", assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'), role_name='eks-master-role', managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name='AdministratorAccess' ) ] ) k8s_instance_profile = iam.CfnInstanceProfile(self, 'instanceprofile', roles=[k8s_admin.role_name], instance_profile_name='eks-master-role' ) cluster = eks.Cluster(self, 'dev', cluster_name='eks-cdk-demo', version='1.15', vpc=vpc, vpc_subnets=[ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE)], default_capacity=0, kubectl_enabled=True, #security_group=k8s_sg, masters_role= k8s_admin ) #cluster.aws_auth.add_user_mapping(adminuser, {groups['system:masters']}) ng = cluster.add_nodegroup('eks-ng', nodegroup_name='eks-ng', instance_type=ec2.InstanceType('t3.medium'), disk_size=5, min_size=1, max_size=1, desired_size=1, subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE), remote_access=eks.NodegroupRemoteAccess(ssh_key_name='k8s-nodes') )
def __init__(self, scope: core.Construct, construct_id: str, vpc: ec2.Vpc, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # The code that defines your stack goes here env_name = self.node.try_get_context("env") eks_role = iam.Role( self, "eksadmin", assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'), role_name='eks-cluster-role', managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name='AdministratorAccess') ]) eks_instance_profile = iam.CfnInstanceProfile( self, 'instanceprofile', roles=[eks_role.role_name], instance_profile_name='eks-cluster-role') cluster = eks.Cluster( self, 'prod', cluster_name='ie-prod-snow-common', version=eks.KubernetesVersion.V1_19, vpc=vpc, vpc_subnets=[ ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE) ], default_capacity=0, masters_role=eks_role) nodegroup = cluster.add_nodegroup_capacity( 'eks-nodegroup', instance_types=[ ec2.InstanceType('t3.large'), ec2.InstanceType('m5.large'), ec2.InstanceType('c5.large') ], disk_size=50, min_size=2, max_size=2, desired_size=2, subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE), remote_access=eks.NodegroupRemoteAccess( ssh_key_name='ie-prod-snow-common'), capacity_type=eks.CapacityType.SPOT)
def __init__(self, scope: core.Construct, id: str, *, role_name_prefix: Optional[str] = None, artifacts_bucket: Optional[s3.Bucket] = None, artifacts_path: Optional[str] = None, logs_bucket: Optional[s3.Bucket] = None, logs_path: Optional[str] = None) -> None: super().__init__(scope, id) if role_name_prefix: self._service_role = EMRRoles._create_service_role( self, 'EMRServiceRole', role_name='{}-ServiceRole'.format(role_name_prefix)) self._instance_role = EMRRoles._create_instance_role( self, 'EMRInstanceRole', role_name='{}-InstanceRole'.format(role_name_prefix)) self._autoscaling_role = EMRRoles._create_autoscaling_role( self, 'EMRAutoScalingRole', role_name='{}-AutoScalingRole'.format(role_name_prefix)) self._instance_profile = iam.CfnInstanceProfile( self, '{}_InstanceProfile'.format(id), roles=[self._instance_role.role_name], instance_profile_name=self._instance_role.role_name) self._instance_profile_arn = self._instance_profile.attr_arn if artifacts_bucket: artifacts_bucket.grant_read( self._instance_role, os.path.join(artifacts_path, '*') if artifacts_path else artifacts_path).assert_success() if logs_bucket: logs_bucket.grant_read_write( self._service_role, os.path.join(logs_path, '*') if logs_path else logs_path).assert_success() logs_bucket.grant_read_write( self._instance_role, os.path.join(logs_path, '*') if logs_path else logs_path).assert_success()
def create_emr_instance_role(scope: core.Construct) -> iam.Role: """ create an IAM (service) role for all EC2 instances (the service principal ec2) in our cluster the role is assumed by nodes that run insied of EMR/Hadoop and hence they are allowed to talk to other AWS services under the hood this does pretty much the same as if someone would click the default roles in AWS EMR-UI https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-iam-role-for-ec2.html """ instance_role = iam.Role( scope, id=f"EmrInstanceRole", assumed_by=iam.ServicePrincipal(f"ec2.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonElasticMapReduceforEC2Role") ], ) instance_profile = iam.CfnInstanceProfile(scope, "EmrInstanceProfile", roles=[instance_role.role_name]) return iam.Role.from_role_arn(scope, "EmrInstaceProfileRole", instance_profile.attr_arn)
def __init__(self, app: core.Construct, stack_name: str, vpc: aws_ec2.Vpc, security_group: aws_ec2.SecurityGroup): super().__init__(scope=app, id=f"{stack_name}-batch") batch_role = aws_iam.Role( scope=self, id=f"batch_role", role_name=f"batch_role", assumed_by=aws_iam.ServicePrincipal("batch.amazonaws.com")) batch_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AWSBatchServiceRole", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole")) batch_role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["arn:aws:logs:*:*:*"], actions=[ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DescribeLogStreams" ])) # Role to attach EC2 instance_role = aws_iam.Role( scope=self, id=f"instance_role", role_name=f"instance_role_for", assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com")) instance_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AmazonEC2ContainerServiceforEC2Role", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" )) # add policy to access S3 instance_role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["s3:*"])) # add policy to access CloudWatch Logs instance_role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["arn:aws:logs:*:*:*"], actions=[ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DescribeLogStreams" ])) # attach role to EC2 instance_profile = aws_iam.CfnInstanceProfile( scope=self, id=f"instance_profile", instance_profile_name=f"instance_profile", roles=[instance_role.role_name]) # ===== # # batch # # ===== # batch_compute_resources = aws_batch.ComputeResources( vpc=vpc, maxv_cpus=4, minv_cpus=0, security_groups=[security_group], instance_role=instance_profile.attr_arn, type=aws_batch.ComputeResourceType.SPOT) batch_compute_environment = aws_batch.ComputeEnvironment( scope=self, id="batch_compute_environment", compute_environment_name="batch_compute_environment", compute_resources=batch_compute_resources, service_role=batch_role) job_role = aws_iam.Role( scope=self, id=f"job_role", role_name=f"job_role", assumed_by=aws_iam.ServicePrincipal("ecs-tasks.amazonaws.com")) job_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AmazonECSTaskExecutionRolePolicy", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" )) job_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AmazonS3FullAccess", managed_policy_arn="arn:aws:iam::aws:policy/AmazonS3FullAccess" )) job_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"CloudWatchLogsFullAccess", managed_policy_arn= "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")) self.batch_job_queue = aws_batch.JobQueue( scope=self, id=f"job_queue", job_queue_name=f"job_queue", compute_environments=[ aws_batch.JobQueueComputeEnvironment( compute_environment=batch_compute_environment, order=1) ], priority=1) # ECR repository ecr_repository = aws_ecr_assets.DockerImageAsset( scope=self, id=f"ecr_image", directory="./docker", repository_name=f"repository") # get image from ECR container_image = aws_ecs.ContainerImage.from_ecr_repository( repository=ecr_repository.repository) # job define # pass `S3_BUCKET` as environment argument. self.batch_job_definition = aws_batch.JobDefinition( scope=self, id=f"job_definition", job_definition_name=f"job_definition", container=aws_batch.JobDefinitionContainer( image=container_image, environment={"S3_BUCKET": f"{S3_BUCKET}"}, job_role=job_role, vcpus=1, memory_limit_mib=1024))
def __init__(self, scope: core.Construct, id: str, config_dict, **kwargs) -> None: super().__init__(scope, id, **kwargs) """ Get VPC details """ vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_id=config_dict['vpc_id']) """ Create Security Group for Batch Env """ batch_security_group = "datalake-batch-security-group" createBatchSecurityGroup = ec2.SecurityGroup( self, "createBatchSecurityGroup", vpc=vpc, allow_all_outbound=True, description= "This security group will be used for AWS Batch Compute Env", security_group_name=batch_security_group) createBatchSecurityGroup.add_ingress_rule( peer=ec2.Peer.ipv4("0.0.0.0/0"), connection=ec2.Port(protocol=ec2.Protocol.TCP, string_representation="ingress_rule", from_port=22, to_port=22)) createBatchSecurityGroup.add_egress_rule( peer=ec2.Peer.ipv4("0.0.0.0/0"), connection=ec2.Port(protocol=ec2.Protocol.TCP, string_representation="egress_rule", from_port=-1, to_port=-1)) core.CfnOutput(self, "createBatchSecurityGroupId", value=createBatchSecurityGroup.security_group_id) """ Create IAM Role for ecsInstance """ createECSInstanceRole = iam.Role( self, "createECSInstanceRole", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), description= "This instance role will be used by the ECS cluster instances", managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonEC2FullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "AWSBatchFullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "SecretsManagerReadWrite"), iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonAthenaFullAccess"), iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/" "AmazonEC2ContainerServiceforEC2Role"), iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSBatchServiceRole") ], role_name="datalake-ecsinstance-role") createInstanceProfile = iam.CfnInstanceProfile( self, "createInstanceProfile", roles=[createECSInstanceRole.role_name], instance_profile_name="datalake-ecsinstance-role") useECSInstanceProfile = createInstanceProfile.instance_profile_name core.CfnOutput(self, "createECSInstanceRoleName", value=createECSInstanceRole.role_name) """ Create Spot Fleet Role """ createSpotFleetRole = iam.Role( self, 'createSpotFleetRole', assumed_by=iam.ServicePrincipal("spotfleet.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2SpotFleetTaggingRole") ]) core.CfnOutput(self, "createSpotFleetRoleName", value=createSpotFleetRole.role_name) useSpotFleetRole = createSpotFleetRole.without_policy_updates() """ Create Batch Service Role """ createBatchServiceRole = iam.Role( self, 'createBatchServiceRole', assumed_by=iam.ServicePrincipal("batch.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSBatchServiceRole") ]) core.CfnOutput(self, "createBatchServiceRoleName", value=createBatchServiceRole.role_name) useBatchServiceRole = createBatchServiceRole.without_policy_updates() """ Create Compute Environment """ subnet_1 = ec2.Subnet.from_subnet_attributes( self, "subnet_1", subnet_id=config_dict['SubnetIds'].split(",")[0], availability_zone=config_dict['AvailabilityZones'].split(",")[0]) subnet_2 = ec2.Subnet.from_subnet_attributes( self, "subnet_2", subnet_id=config_dict['SubnetIds'].split(",")[1], availability_zone=config_dict['AvailabilityZones'].split(",")[1]) createBatchComputeEnv = batch.ComputeEnvironment( self, "createBatchComputeEnv", compute_environment_name="datalake-compute-env", service_role=useBatchServiceRole, compute_resources=batch.ComputeResources( vpc=vpc, type=batch.ComputeResourceType.SPOT, bid_percentage=60, desiredv_cpus=0, maxv_cpus=100, minv_cpus=0, security_groups=[createBatchSecurityGroup], vpc_subnets=ec2.SubnetSelection(subnets=[subnet_1, subnet_2]), instance_role=useECSInstanceProfile, spot_fleet_role=useSpotFleetRole, compute_resources_tags=core.Tag.add( self, 'Name', 'Datalake Pipeline Instance'))) core.CfnOutput(self, "createBatchComputeEnvName", value=createBatchComputeEnv.compute_environment_name) getIComputeEnvObject = batch.ComputeEnvironment.from_compute_environment_arn( self, "getComputeEnvAtrributes", compute_environment_arn=createBatchComputeEnv. compute_environment_arn) """ Create Batch Job Queue """ createBatchJobQueue = batch.JobQueue( self, "createBatchJobQueue", compute_environments=[ batch.JobQueueComputeEnvironment( compute_environment=getIComputeEnvObject, order=1) ], enabled=True, job_queue_name="datalake-job-queue", priority=1) core.CfnOutput(self, "createBatchJobQueueName", value=createBatchJobQueue.job_queue_name) """ Create ECR Repo for datalake images """ createECRRepo = ecr.Repository( self, "createECRRepo", repository_name=config_dict['workflow_ecr_repo']) core.CfnOutput(self, "createECRRepoName", value=createECRRepo.repository_name)
def __init__(self, scope: core.Stack, id: str, **kwargs): super().__init__(scope, id, **kwargs) # This resource alone will create a private/public subnet in each AZ as well as nat/internet gateway(s) self.vpc = aws_ec2.Vpc( self, "BaseVPC", cidr='10.0.0.0/24', ) # Creating ECS Cluster in the VPC created above self.ecs_cluster = aws_ecs.Cluster(self, "ECSCluster", vpc=self.vpc, cluster_name="container-demo") # Adding service discovery namespace to cluster self.ecs_cluster.add_default_cloud_map_namespace(name="service", ) ###### EC2 SPOT CAPACITY PROVIDER SECTION ###### ## As of today, AWS CDK doesn't support Launch Templates on the AutoScaling construct, hence it ## doesn't support Mixed Instances Policy to combine instance types on Auto Scaling and adhere to Spot best practices ## In the meantime, CfnLaunchTemplate and CfnAutoScalingGroup resources are used to configure Spot capacity ## https://github.com/aws/aws-cdk/issues/6734 self.ecs_spot_instance_role = aws_iam.Role( self, "ECSSpotECSInstanceRole", assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com"), managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2ContainerServiceforEC2Role"), aws_iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2RoleforSSM") ]) self.ecs_spot_instance_profile = aws_iam.CfnInstanceProfile( self, "ECSSpotInstanceProfile", roles=[self.ecs_spot_instance_role.role_name]) ## This creates a Launch Template for the Auto Scaling group self.lt = aws_ec2.CfnLaunchTemplate( self, "ECSEC2SpotCapacityLaunchTemplate", launch_template_data={ "instanceType": "m5.large", "imageId": aws_ssm.StringParameter.value_for_string_parameter( self, "/aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id" ), "securityGroupIds": [ x.security_group_id for x in self.ecs_cluster.connections.security_groups ], "iamInstanceProfile": { "arn": self.ecs_spot_instance_profile.attr_arn }, # ## Here we configure the ECS agent to drain Spot Instances upon catching a Spot Interruption notice from instance metadata "userData": core.Fn.base64( core.Fn.sub( "#!/usr/bin/bash\n" "echo ECS_CLUSTER=${cluster_name} >> /etc/ecs/ecs.config\n" "sudo iptables --insert FORWARD 1 --in-interface docker+ --destination 169.254.169.254/32 --jump DROP\n" "sudo service iptables save\n" "echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true >> /etc/ecs/ecs.config\n" "echo ECS_AWSVPC_BLOCK_IMDS=true >> /etc/ecs/ecs.config\n" "cat /etc/ecs/ecs.config", variables={ "cluster_name": self.ecs_cluster.cluster_name })) }, launch_template_name="ECSEC2SpotCapacityLaunchTemplate") self.ecs_ec2_spot_mig_asg = aws_autoscaling.CfnAutoScalingGroup( self, "ECSEC2SpotCapacity", min_size="0", max_size="10", vpc_zone_identifier=[ x.subnet_id for x in self.vpc.private_subnets ], mixed_instances_policy={ "instancesDistribution": { "onDemandAllocationStrategy": "prioritized", "onDemandBaseCapacity": 0, "onDemandPercentageAboveBaseCapacity": 0, "spotAllocationStrategy": "capacity-optimized" }, "launchTemplate": { "launchTemplateSpecification": { "launchTemplateId": self.lt.ref, "version": self.lt.attr_default_version_number }, "overrides": [{ "instanceType": "m5.large" }, { "instanceType": "m5d.large" }, { "instanceType": "m5a.large" }, { "instanceType": "m5ad.large" }, { "instanceType": "m5n.large" }, { "instanceType": "m5dn.large" }, { "instanceType": "m3.large" }, { "instanceType": "m4.large" }, { "instanceType": "t3.large" }, { "instanceType": "t2.large" }] } }) # core.Tag.add(self.ecs_ec2_spot_mig_asg, "Name", self.ecs_ec2_spot_mig_asg.node.path) core.CfnOutput(self, "EC2SpotAutoScalingGroupName", value=self.ecs_ec2_spot_mig_asg.ref, export_name="EC2SpotASGName") # ##### END EC2 SPOT CAPACITY PROVIDER SECTION ##### # Namespace details as CFN output self.namespace_outputs = { 'ARN': self.ecs_cluster.default_cloud_map_namespace. private_dns_namespace_arn, 'NAME': self.ecs_cluster.default_cloud_map_namespace. private_dns_namespace_name, 'ID': self.ecs_cluster.default_cloud_map_namespace. private_dns_namespace_id, } # Cluster Attributes self.cluster_outputs = { 'NAME': self.ecs_cluster.cluster_name, 'SECGRPS': str(self.ecs_cluster.connections.security_groups) } # When enabling EC2, we need the security groups "registered" to the cluster for imports in other service stacks if self.ecs_cluster.connections.security_groups: self.cluster_outputs['SECGRPS'] = str([ x.security_group_id for x in self.ecs_cluster.connections.security_groups ][0]) # Frontend service to backend services on 3000 self.services_3000_sec_group = aws_ec2.SecurityGroup( self, "FrontendToBackendSecurityGroup", allow_all_outbound=True, description= "Security group for frontend service to talk to backend services", vpc=self.vpc) # Allow inbound 3000 from ALB to Frontend Service self.sec_grp_ingress_self_3000 = aws_ec2.CfnSecurityGroupIngress( self, "InboundSecGrp3000", ip_protocol='TCP', source_security_group_id=self.services_3000_sec_group. security_group_id, from_port=3000, to_port=3000, group_id=self.services_3000_sec_group.security_group_id) # Creating an EC2 bastion host to perform load test on private backend services amzn_linux = aws_ec2.MachineImage.latest_amazon_linux( generation=aws_ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=aws_ec2.AmazonLinuxEdition.STANDARD, virtualization=aws_ec2.AmazonLinuxVirt.HVM, storage=aws_ec2.AmazonLinuxStorage.GENERAL_PURPOSE) # Instance Role/profile that will be attached to the ec2 instance # Enabling service role so the EC2 service can use ssm role = aws_iam.Role( self, "InstanceSSM", assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com")) # Attaching the SSM policy to the role so we can use SSM to ssh into the ec2 instance role.add_managed_policy( aws_iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2RoleforSSM")) # Reading user data, to install siege into the ec2 instance. with open("stresstool_user_data.sh") as f: user_data = f.read() # Instance creation self.instance = aws_ec2.Instance( self, "Instance", instance_name="{}-stresstool".format(stack_name), instance_type=aws_ec2.InstanceType("t3.medium"), machine_image=amzn_linux, vpc=self.vpc, role=role, user_data=aws_ec2.UserData.custom(user_data), security_group=self.services_3000_sec_group) # All Outputs required for other stacks to build core.CfnOutput(self, "NSArn", value=self.namespace_outputs['ARN'], export_name="NSARN") core.CfnOutput(self, "NSName", value=self.namespace_outputs['NAME'], export_name="NSNAME") core.CfnOutput(self, "NSId", value=self.namespace_outputs['ID'], export_name="NSID") core.CfnOutput(self, "FE2BESecGrp", value=self.services_3000_sec_group.security_group_id, export_name="SecGrpId") core.CfnOutput(self, "ECSClusterName", value=self.cluster_outputs['NAME'], export_name="ECSClusterName") core.CfnOutput(self, "ECSClusterSecGrp", value=self.cluster_outputs['SECGRPS'], export_name="ECSSecGrpList") core.CfnOutput(self, "ServicesSecGrp", value=self.services_3000_sec_group.security_group_id, export_name="ServicesSecGrp") core.CfnOutput(self, "StressToolEc2Id", value=self.instance.instance_id) core.CfnOutput(self, "StressToolEc2Ip", value=self.instance.instance_private_ip)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) eks_vpc = ec2.Vpc(self, "VPC", cidr="10.0.0.0/16") self.node.apply_aspect( core.Tag("kubernetes.io/cluster/cluster", "shared")) eks_vpc.private_subnets[0].node.apply_aspect( core.Tag("kubernetes.io/role/internal-elb", "1")) eks_vpc.private_subnets[1].node.apply_aspect( core.Tag("kubernetes.io/role/internal-elb", "1")) eks_vpc.public_subnets[0].node.apply_aspect( core.Tag("kubernetes.io/role/elb", "1")) eks_vpc.public_subnets[1].node.apply_aspect( core.Tag("kubernetes.io/role/elb", "1")) # Create IAM Role For CodeBuild and Cloud9 codebuild_role = iam.Role( self, "BuildRole", assumed_by=iam.CompositePrincipal( iam.ServicePrincipal("codebuild.amazonaws.com"), iam.ServicePrincipal("ec2.amazonaws.com")), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "AdministratorAccess") ]) instance_profile = iam.CfnInstanceProfile( self, "InstanceProfile", roles=[codebuild_role.role_name]) # Create CodeBuild PipelineProject build_project = codebuild.PipelineProject( self, "BuildProject", role=codebuild_role, build_spec=codebuild.BuildSpec.from_source_filename( "buildspec.yml")) # Create CodePipeline pipeline = codepipeline.Pipeline( self, "Pipeline", ) # Create Artifact artifact = codepipeline.Artifact() # S3 Source Bucket source_bucket = s3.Bucket.from_bucket_attributes( self, "SourceBucket", bucket_arn=core.Fn.join( "", ["arn:aws:s3:::ee-assets-prod-", core.Fn.ref("AWS::Region")])) # Add Source Stage pipeline.add_stage( stage_name="Source", actions=[ codepipeline_actions.S3SourceAction( action_name="S3SourceRepo", bucket=source_bucket, bucket_key= "modules/2cae1f20008d4fc5aaef294602649b98/v9/source.zip", output=artifact, trigger=codepipeline_actions.S3Trigger.NONE) ]) # Add CodeBuild Stage pipeline.add_stage( stage_name="Deploy", actions=[ codepipeline_actions.CodeBuildAction( action_name="CodeBuildProject", project=build_project, type=codepipeline_actions.CodeBuildActionType.BUILD, input=artifact, environment_variables={ 'PublicSubnet1ID': codebuild.BuildEnvironmentVariable( value=eks_vpc.public_subnets[0].subnet_id), 'PublicSubnet2ID': codebuild.BuildEnvironmentVariable( value=eks_vpc.public_subnets[1].subnet_id), 'PrivateSubnet1ID': codebuild.BuildEnvironmentVariable( value=eks_vpc.private_subnets[0].subnet_id), 'PrivateSubnet2ID': codebuild.BuildEnvironmentVariable( value=eks_vpc.private_subnets[1].subnet_id), 'AWS_DEFAULT_REGION': codebuild.BuildEnvironmentVariable(value=self.region), 'INSTANCEPROFILEID': codebuild.BuildEnvironmentVariable( value=instance_profile.ref), 'AWS_ACCOUNT_ID': codebuild.BuildEnvironmentVariable(value=self.account) }) ]) cloud9_stack = cloudformation.CfnStack( self, "Cloud9Stack", # template_url="https://aws-quickstart.s3.amazonaws.com/quickstart-cloud9-ide/templates/cloud9-ide-instance.yaml", template_url= "https://ee-assets-prod-us-east-1.s3.amazonaws.com/modules/2cae1f20008d4fc5aaef294602649b98/v9/cloud9-ide-instance.yaml", parameters={ "C9InstanceType": "m5.large", "C9Subnet": eks_vpc.public_subnets[0].subnet_id }) pipeline.node.add_dependency(eks_vpc) pipeline.node.add_dependency(cloud9_stack)
def __init__(self, scope: cdk.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### Parameters bootstrap_script_args = cdk.CfnParameter(self, 'BootstrapScriptArgs', type='String', default='', description='Space seperated arguments passed to the bootstrap script.' ) # create a VPC vpc = ec2.Vpc(self, 'VPC', cidr='10.0.0.0/16', max_azs=99) # create a private and public subnet per vpc selection = vpc.select_subnets( subnet_type=ec2.SubnetType.PRIVATE ) # Output created subnets for i, public_subnet in enumerate(vpc.public_subnets): cdk.CfnOutput(self, 'PublicSubnet%i' % i, value=public_subnet.subnet_id) for i, private_subnet in enumerate(vpc.private_subnets): cdk.CfnOutput(self, 'PrivateSubnet%i' % i, value=private_subnet.subnet_id) cdk.CfnOutput(self, 'VPCId', value=vpc.vpc_id) # Create a Bucket bucket = s3.Bucket(self, "DataRepository") quickstart_bucket = s3.Bucket.from_bucket_name(self, 'QuickStartBucket', 'aws-quickstart') # Upload Bootstrap Script to that bucket bootstrap_script = assets.Asset(self, 'BootstrapScript', path='scripts/bootstrap.sh' ) # Upload parallel cluster post_install_script to that bucket pcluster_post_install_script = assets.Asset(self, 'PclusterPostInstallScript', path='scripts/post_install_script.sh' ) # Setup CloudTrail cloudtrail.Trail(self, 'CloudTrail', bucket=bucket) # Create a Cloud9 instance # Cloud9 doesn't have the ability to provide userdata # Because of this we need to use SSM run command cloud9_instance = cloud9.Ec2Environment(self, 'Cloud9Env', vpc=vpc, instance_type=ec2.InstanceType(instance_type_identifier='c5.large')) cdk.CfnOutput(self, 'URL', value=cloud9_instance.ide_url) # Create a keypair in lambda and store the private key in SecretsManager c9_createkeypair_role = iam.Role(self, 'Cloud9CreateKeypairRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) c9_createkeypair_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole')) # Add IAM permissions to the lambda role c9_createkeypair_role.add_to_policy(iam.PolicyStatement( actions=[ 'ec2:CreateKeyPair', 'ec2:DeleteKeyPair' ], resources=['*'], )) # Lambda for Cloud9 keypair c9_createkeypair_lambda = _lambda.Function(self, 'C9CreateKeyPairLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(300), role=c9_createkeypair_role, code=_lambda.Code.asset('functions/source/c9keypair'), # code=_lambda.Code.from_bucket( ) c9_createkeypair_provider = cr.Provider(self, "C9CreateKeyPairProvider", on_event_handler=c9_createkeypair_lambda) c9_createkeypair_cr = cfn.CustomResource(self, "C9CreateKeyPair", provider=c9_createkeypair_provider, properties={ 'ServiceToken': c9_createkeypair_lambda.function_arn } ) #c9_createkeypair_cr.node.add_dependency(instance_id) c9_ssh_private_key_secret = secretsmanager.CfnSecret(self, 'SshPrivateKeySecret', secret_string=c9_createkeypair_cr.get_att_string('PrivateKey') ) # The iam policy has a <REGION> parameter that needs to be replaced. # We do it programmatically so future versions of the synth'd stack # template include all regions. with open('iam/ParallelClusterUserPolicy.json') as json_file: data = json.load(json_file) for s in data['Statement']: if s['Sid'] == 'S3ParallelClusterReadOnly': s['Resource'] = [] for r in region_info.RegionInfo.regions: s['Resource'].append('arn:aws:s3:::{0}-aws-parallelcluster*'.format(r.name)) parallelcluster_user_policy = iam.CfnManagedPolicy(self, 'ParallelClusterUserPolicy', policy_document=iam.PolicyDocument.from_json(data)) # Cloud9 IAM Role cloud9_role = iam.Role(self, 'Cloud9Role', assumed_by=iam.ServicePrincipal('ec2.amazonaws.com')) cloud9_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('AmazonSSMManagedInstanceCore')) cloud9_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('AWSCloud9User')) cloud9_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, 'AttachParallelClusterUserPolicy', parallelcluster_user_policy.ref)) cloud9_role.add_to_policy(iam.PolicyStatement( resources=['*'], actions=[ 'ec2:DescribeInstances', 'ec2:DescribeVolumes', 'ec2:ModifyVolume' ] )) cloud9_role.add_to_policy(iam.PolicyStatement( resources=[c9_ssh_private_key_secret.ref], actions=[ 'secretsmanager:GetSecretValue' ] )) bootstrap_script.grant_read(cloud9_role) pcluster_post_install_script.grant_read(cloud9_role) # Cloud9 User # user = iam.User(self, 'Cloud9User', password=cdk.SecretValue.plain_text('supersecretpassword'), password_reset_required=True) # Cloud9 Setup IAM Role cloud9_setup_role = iam.Role(self, 'Cloud9SetupRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com')) cloud9_setup_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSLambdaBasicExecutionRole')) # Allow pcluster to be run in bootstrap cloud9_setup_role.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, 'AttachParallelClusterUserPolicySetup', parallelcluster_user_policy.ref)) # Add IAM permissions to the lambda role cloud9_setup_role.add_to_policy(iam.PolicyStatement( actions=[ 'cloudformation:DescribeStackResources', 'ec2:AssociateIamInstanceProfile', 'ec2:AuthorizeSecurityGroupIngress', 'ec2:DescribeInstances', 'ec2:DescribeInstanceStatus', 'ec2:DescribeInstanceAttribute', 'ec2:DescribeIamInstanceProfileAssociations', 'ec2:DescribeVolumes', 'ec2:DesctibeVolumeAttribute', 'ec2:DescribeVolumesModifications', 'ec2:DescribeVolumeStatus', 'ssm:DescribeInstanceInformation', 'ec2:ModifyVolume', 'ec2:ReplaceIamInstanceProfileAssociation', 'ec2:ReportInstanceStatus', 'ssm:SendCommand', 'ssm:GetCommandInvocation', 's3:GetObject', 'lambda:AddPermission', 'lambda:RemovePermission', 'events:PutRule', 'events:DeleteRule', 'events:PutTargets', 'events:RemoveTargets', ], resources=['*'], )) cloud9_setup_role.add_to_policy(iam.PolicyStatement( actions=['iam:PassRole'], resources=[cloud9_role.role_arn] )) cloud9_setup_role.add_to_policy(iam.PolicyStatement( actions=[ 'lambda:AddPermission', 'lambda:RemovePermission' ], resources=['*'] )) # Cloud9 Instance Profile c9_instance_profile = iam.CfnInstanceProfile(self, "Cloud9InstanceProfile", roles=[cloud9_role.role_name]) # Lambda to add Instance Profile to Cloud9 c9_instance_profile_lambda = _lambda.Function(self, 'C9InstanceProfileLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(900), role=cloud9_setup_role, code=_lambda.Code.asset('functions/source/c9InstanceProfile'), ) c9_instance_profile_provider = cr.Provider(self, "C9InstanceProfileProvider", on_event_handler=c9_instance_profile_lambda, ) instance_id = cfn.CustomResource(self, "C9InstanceProfile", provider=c9_instance_profile_provider, properties={ 'InstanceProfile': c9_instance_profile.ref, 'Cloud9Environment': cloud9_instance.environment_id, } ) instance_id.node.add_dependency(cloud9_instance) # Lambda for Cloud9 Bootstrap c9_bootstrap_lambda = _lambda.Function(self, 'C9BootstrapLambda', runtime=_lambda.Runtime.PYTHON_3_6, handler='lambda_function.handler', timeout=cdk.Duration.seconds(900), role=cloud9_setup_role, code=_lambda.Code.asset('functions/source/c9bootstrap'), ) c9_bootstrap_provider = cr.Provider(self, "C9BootstrapProvider", on_event_handler=c9_bootstrap_lambda) c9_bootstrap_cr = cfn.CustomResource(self, "C9Bootstrap", provider=c9_bootstrap_provider, properties={ 'Cloud9Environment': cloud9_instance.environment_id, 'BootstrapPath': 's3://%s/%s' % (bootstrap_script.s3_bucket_name, bootstrap_script.s3_object_key), 'BootstrapArguments': bootstrap_script_args, 'VPCID': vpc.vpc_id, 'MasterSubnetID': vpc.public_subnets[0].subnet_id, 'ComputeSubnetID': vpc.private_subnets[0].subnet_id, 'PostInstallScriptS3Url': "".join( ['s3://', pcluster_post_install_script.s3_bucket_name, "/", pcluster_post_install_script.s3_object_key ] ), 'PostInstallScriptBucket': pcluster_post_install_script.s3_bucket_name, 'KeyPairId': c9_createkeypair_cr.ref, 'KeyPairSecretArn': c9_ssh_private_key_secret.ref } ) c9_bootstrap_cr.node.add_dependency(instance_id) c9_bootstrap_cr.node.add_dependency(c9_createkeypair_cr) c9_bootstrap_cr.node.add_dependency(c9_ssh_private_key_secret)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # get vpc used vpc-id vpc = ec2.Vpc.from_lookup(self, "EKSNode", vpc_id=vpc_id) subnets = [] if len(vpc.private_subnets) > 0: subnets.extend(vpc.private_subnets) elif len(vpc.public_subnets) > 0: subnets.extend(vpc.public_subnets) else: print("Not any subnets found,") return # add a worker node role workerRole = iam.Role.from_role_arn(self, 'nodeRole', role_arn=nodes_role_arn) #node_sg=ec2.SecurityGroup.from_security_group_id(self,"nodeSG",security_group_id='sg-0461d7bdfb942d0ef') # add iam instance profile instanceProfile = iam.CfnInstanceProfile( self, 'kopsNodeProfile', roles=[workerRole.role_name], instance_profile_name='eks-cluster-workerNodeProfile') # read and base64 encode userdata file data = open('nodeup.sh', 'rb').read() encodedBytes = base64.encodebytes(data) encodedStr = str(encodedBytes, "utf-8") # add worker instances and associate EIPs for i in range(0, 1): # add a network interface eni0 = ec2.CfnNetworkInterface( self, 'eni-' + str(i), subnet_id='subnet-040455b57b16a4cc9', group_set=['sg-0461d7bdfb942d0ef', 'sg-0b3ae225b27e04679']) # add worker instances instance = ec2.CfnInstance( self, "kops-node-" + str(i), image_id=ami_id, instance_type="t2.medium", block_device_mappings=[ { 'deviceName': '/dev/xvda', 'ebs': { 'deleteOnTermination': True, 'volumeSize': 40, 'volumeType': 'gp2', 'encrypted': False, }, }, { 'deviceName': '/dev/sdb', # for /var/lib/docker 'ebs': { 'deleteOnTermination': True, 'volumeSize': 100, 'volumeType': 'gp2', 'encrypted': False, }, }, { 'deviceName': '/dev/sdc', # for data volume 'ebs': { 'deleteOnTermination': True, 'volumeSize': 200, 'volumeType': 'gp2', 'encrypted': False, }, }, ], key_name=key_name, network_interfaces=[{ 'deviceIndex': '0', 'networkInterfaceId': eni0.ref, }], iam_instance_profile=instanceProfile.ref, #iam_instance_profile=nodes_role_arn, tags=[ core.CfnTag(key="KubernetesCluster", value="eks-asgfleet-01"), core.CfnTag(key="Name", value="test-01"), core.CfnTag(key="k8s.io/role/node", value="1"), core.CfnTag(key="CDK/manual", value="singlenode"), ], user_data=encodedStr) #associate EIP with the instance eip = ec2.CfnEIP(self, "eip-" + str(i)) ec2.CfnEIPAssociation(self, "eip-ass-i" + str(i), allocation_id=eip.attr_allocation_id, network_interface_id=eni0.ref)
def __init__(self, app: cdk.App, id: str, vpc: ec2.Vpc, servicedomain: str, **kwargs) -> None: super().__init__(app, id) cluster = ecs.Cluster(self, id, vpc=vpc) cluster.add_default_cloud_map_namespace( name=servicedomain, type=ecs.NamespaceType.PrivateDns) self._cluster = cluster ecssg = ec2.SecurityGroup(self, 'ECSServiceSecurityGroup', vpc=vpc) ecssg.add_ingress_rule(peer=ec2.CidrIPv4(vpc.vpc_cidr_block), connection=ec2.TcpAllPorts()) self._clustersg = ecssg # Bastion host stuff ------------------------------------------------------------------------------------- # BastionInstanceRole pd = pu.PolicyUtils.createpolicyfromfile( './appmeshdemo/policydocs/appmesh.json') bir = iam.Role( self, 'BastionInstanceRole', assumed_by=iam.ServicePrincipal('ec2'), inline_policies={'appmesh': pd}, managed_policy_arns=[ 'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM' ]) bip = iam.CfnInstanceProfile(self, 'BastionInstanceProfile', roles=[bir.role_name]) # Bastion EC2 instance bsg = ec2.SecurityGroup(self, 'BastionSG', vpc=vpc) bsg.add_ingress_rule(peer=ec2.AnyIPv4(), connection=ec2.TcpAllPorts()) ni = ec2.CfnNetworkInterfaceProps() ni['associatePublicIpAddress'] = True ni['deviceIndex'] = '0' ni['groupSet'] = [bsg.security_group_name] ni['subnetId'] = vpc.public_subnets[0].subnet_id bhi = ec2.CfnInstance( self, 'BastionInstance', instance_type='t2.micro', iam_instance_profile=bip.instance_profile_name, image_id=ec2.AmazonLinuxImage().get_image(self).image_id, network_interfaces=[ni]) # Load-Balancer stuff ------------------------------------------------------------------------------------ plbsg = ec2.SecurityGroup(self, 'PublicLoadBalancerSG', vpc=vpc) plbsg.add_ingress_rule(peer=ec2.AnyIPv4(), connection=ec2.TcpPortRange(0, 65535)) plb = elbv2.ApplicationLoadBalancer(self, 'PublicLoadBalancer', internet_facing=True, load_balancer_name='appmeshdemo', security_group=plbsg, vpc=vpc, idle_timeout_secs=30) self._publoadbal = plb healthchk = elbv2.HealthCheck() healthchk['intervalSecs'] = 6 healthchk['healthyThresholdCount'] = 2 healthchk['unhealthyThresholdCount'] = 2 dtg = elbv2.ApplicationTargetGroup( self, 'DummyTargetGroupPublic', vpc=vpc, port=80, protocol=elbv2.ApplicationProtocol.Http, health_check=healthchk, target_group_name='appmeshdemo-drop-1') plbl = elbv2.ApplicationListener( self, 'PublicLoadBalancerListener', load_balancer=plb, port=80, protocol=elbv2.ApplicationProtocol.Http, default_target_groups=[dtg]) cdk.CfnOutput(self, id='External URL', value='http://' + plb.load_balancer_dns_name)
def __init__( self, scope: Construct, construct_id: str, *, deploy_env: str, processing_assets_table: aws_dynamodb.Table, ): # pylint: disable=too-many-locals super().__init__(scope, construct_id) if deploy_env == "prod": instance_types = [ aws_ec2.InstanceType("c5.xlarge"), aws_ec2.InstanceType("c5.2xlarge"), aws_ec2.InstanceType("c5.4xlarge"), aws_ec2.InstanceType("c5.9xlarge"), ] else: instance_types = [ aws_ec2.InstanceType("m5.large"), aws_ec2.InstanceType("m5.xlarge"), ] ec2_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonEC2ContainerServiceforEC2Role") batch_instance_role = aws_iam.Role( self, "batch-instance-role", assumed_by=aws_iam.ServicePrincipal( "ec2.amazonaws.com"), # type: ignore[arg-type] managed_policies=[ec2_policy], ) processing_assets_table.grant_read_write_data( batch_instance_role) # type: ignore[arg-type] batch_instance_profile = aws_iam.CfnInstanceProfile( self, "batch-instance-profile", roles=[batch_instance_role.role_name], ) batch_launch_template_data = textwrap.dedent(""" MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="==MYBOUNDARY==" --==MYBOUNDARY== Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash echo ECS_IMAGE_PULL_BEHAVIOR=prefer-cached >> /etc/ecs/ecs.config --==MYBOUNDARY==-- """) launch_template_data = aws_ec2.CfnLaunchTemplate.LaunchTemplateDataProperty( user_data=Fn.base64(batch_launch_template_data.strip())) cloudformation_launch_template = aws_ec2.CfnLaunchTemplate( self, "batch-launch-template", launch_template_name=f"{deploy_env}-datalake-batch-launch-template", launch_template_data=launch_template_data, ) assert cloudformation_launch_template.launch_template_name is not None launch_template = aws_batch.LaunchTemplateSpecification( launch_template_name=cloudformation_launch_template. launch_template_name) # use existing VPC in LINZ AWS account. # VPC with these tags is required to exist in AWS account before being deployed. # A VPC will not be deployed by this project. vpc = aws_ec2.Vpc.from_lookup( self, "datalake-vpc", tags={ APPLICATION_NAME_TAG_NAME: APPLICATION_NAME, "ApplicationLayer": "networking", }, ) compute_resources = aws_batch.ComputeResources( vpc=vpc, minv_cpus=0, desiredv_cpus=0, maxv_cpus=1000, instance_types=instance_types, instance_role=batch_instance_profile.instance_profile_name, allocation_strategy=aws_batch.AllocationStrategy( "BEST_FIT_PROGRESSIVE"), launch_template=launch_template, ) batch_service_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSBatchServiceRole") service_role = aws_iam.Role( self, "batch-service-role", assumed_by=aws_iam.ServicePrincipal( "batch.amazonaws.com"), # type: ignore[arg-type] managed_policies=[batch_service_policy], ) compute_environment = aws_batch.ComputeEnvironment( self, "compute-environment", compute_resources=compute_resources, service_role=service_role, # type: ignore[arg-type] ) self.job_queue = aws_batch.JobQueue( scope, f"{construct_id}-job-queue", compute_environments=[ aws_batch.JobQueueComputeEnvironment( compute_environment=compute_environment, order=10 # type: ignore[arg-type] ), ], priority=10, )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) if self.node.try_get_context('vpc_type'): validate_cdk_json(self) ES_LOADER_TIMEOUT = 600 ###################################################################### # REGION mapping / ELB & Lambda Arch ###################################################################### elb_id_temp = region_info.FactName.ELBV2_ACCOUNT elb_map_temp = region_info.RegionInfo.region_map(elb_id_temp) region_dict = {} for region in elb_map_temp: # ELB account ID region_dict[region] = {'ElbV2AccountId': elb_map_temp[region]} # Lambda Arch if region in ('us-east-1', 'us-east-2', 'us-west-2', 'ap-south-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'eu-central-1', 'eu-west-1', 'eu-west-2'): region_dict[region]['LambdaArch'] = ( aws_lambda.Architecture.ARM_64.name) else: region_dict[region]['LambdaArch'] = ( aws_lambda.Architecture.X86_64.name) region_mapping = core.CfnMapping( scope=self, id='RegionMap', mapping=region_dict) ###################################################################### # get params ###################################################################### allow_source_address = core.CfnParameter( self, 'AllowedSourceIpAddresses', allowed_pattern=r'^[0-9./\s]*', description='Space-delimited list of CIDR blocks', default='10.0.0.0/8 172.16.0.0/12 192.168.0.0/16') sns_email = core.CfnParameter( self, 'SnsEmail', allowed_pattern=r'^[0-9a-zA-Z@_\-\+\.]*', description=('Input your email as SNS topic, where Amazon ' 'OpenSearch Service will send alerts to'), default='*****@*****.**') geoip_license_key = core.CfnParameter( self, 'GeoLite2LicenseKey', allowed_pattern=r'^[0-9a-zA-Z]{16}$', default='xxxxxxxxxxxxxxxx', description=("If you wolud like to enrich geoip locaiton such as " "IP address's country, get a license key form MaxMind" " and input the key. If you not, keep " "xxxxxxxxxxxxxxxx")) reserved_concurrency = core.CfnParameter( self, 'ReservedConcurrency', default=10, type='Number', description=('Input reserved concurrency. Increase this value if ' 'there are steady logs delay despite no errors')) aes_domain_name = self.node.try_get_context('aes_domain_name') bucket = f'{aes_domain_name}-{core.Aws.ACCOUNT_ID}' s3bucket_name_geo = f'{bucket}-geo' s3bucket_name_log = f'{bucket}-log' s3bucket_name_snapshot = f'{bucket}-snapshot' # organizations / multiaccount org_id = self.node.try_get_context('organizations').get('org_id') org_mgmt_id = self.node.try_get_context( 'organizations').get('management_id') org_member_ids = self.node.try_get_context( 'organizations').get('member_ids') no_org_ids = self.node.try_get_context( 'no_organizations').get('aws_accounts') # Overwrite default S3 bucket name as customer name temp_geo = self.node.try_get_context('s3_bucket_name').get('geo') if temp_geo: s3bucket_name_geo = temp_geo else: print('Using default bucket names') temp_log = self.node.try_get_context('s3_bucket_name').get('log') if temp_log: s3bucket_name_log = temp_log elif org_id or no_org_ids: s3bucket_name_log = f'{aes_domain_name}-{self.account}-log' else: print('Using default bucket names') temp_snap = self.node.try_get_context('s3_bucket_name').get('snapshot') if temp_snap: s3bucket_name_snapshot = temp_snap else: print('Using default bucket names') kms_cmk_alias = self.node.try_get_context('kms_cmk_alias') if not kms_cmk_alias: kms_cmk_alias = 'aes-siem-key' print('Using default key alais') ###################################################################### # deploy VPC when context is defined as using VPC ###################################################################### # vpc_type is 'new' or 'import' or None vpc_type = self.node.try_get_context('vpc_type') if vpc_type == 'new': is_vpc = True vpc_cidr = self.node.try_get_context('new_vpc_nw_cidr_block') subnet_cidr_mask = int( self.node.try_get_context('new_vpc_subnet_cidr_mask')) is_vpc = True # VPC vpc_aes_siem = aws_ec2.Vpc( self, 'VpcAesSiem', cidr=vpc_cidr, max_azs=3, nat_gateways=0, subnet_configuration=[ aws_ec2.SubnetConfiguration( subnet_type=aws_ec2.SubnetType.ISOLATED, name='aes-siem-subnet', cidr_mask=subnet_cidr_mask)]) subnet1 = vpc_aes_siem.isolated_subnets[0] subnets = [{'subnet_type': aws_ec2.SubnetType.ISOLATED}] vpc_subnets = aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.ISOLATED) vpc_aes_siem_opt = vpc_aes_siem.node.default_child.cfn_options vpc_aes_siem_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN for subnet in vpc_aes_siem.isolated_subnets: subnet_opt = subnet.node.default_child.cfn_options subnet_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN elif vpc_type == 'import': vpc_id = self.node.try_get_context('imported_vpc_id') vpc_aes_siem = aws_ec2.Vpc.from_lookup( self, 'VpcAesSiem', vpc_id=vpc_id) subnet_ids = get_subnet_ids(self) subnets = [] for number, subnet_id in enumerate(subnet_ids, 1): obj_id = 'Subenet' + str(number) subnet = aws_ec2.Subnet.from_subnet_id(self, obj_id, subnet_id) subnets.append(subnet) subnet1 = subnets[0] vpc_subnets = aws_ec2.SubnetSelection(subnets=subnets) if vpc_type: is_vpc = True # Security Group sg_vpc_noinbound_aes_siem = aws_ec2.SecurityGroup( self, 'AesSiemVpcNoinboundSecurityGroup', security_group_name='aes-siem-noinbound-vpc-sg', vpc=vpc_aes_siem) sg_vpc_aes_siem = aws_ec2.SecurityGroup( self, 'AesSiemVpcSecurityGroup', security_group_name='aes-siem-vpc-sg', vpc=vpc_aes_siem) sg_vpc_aes_siem.add_ingress_rule( peer=aws_ec2.Peer.ipv4(vpc_aes_siem.vpc_cidr_block), connection=aws_ec2.Port.tcp(443),) sg_vpc_opt = sg_vpc_aes_siem.node.default_child.cfn_options sg_vpc_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN # VPC Endpoint vpc_aes_siem.add_gateway_endpoint( 'S3Endpoint', service=aws_ec2.GatewayVpcEndpointAwsService.S3, subnets=subnets) vpc_aes_siem.add_interface_endpoint( 'SQSEndpoint', security_groups=[sg_vpc_aes_siem], service=aws_ec2.InterfaceVpcEndpointAwsService.SQS,) vpc_aes_siem.add_interface_endpoint( 'KMSEndpoint', security_groups=[sg_vpc_aes_siem], service=aws_ec2.InterfaceVpcEndpointAwsService.KMS,) else: is_vpc = False is_vpc = core.CfnCondition( self, 'IsVpc', expression=core.Fn.condition_equals(is_vpc, True)) """ CloudFormation実行時の条件式の書き方 ClassのBasesが aws_cdk.core.Resource の時は、 node.default_child.cfn_options.condition = is_vpc ClassのBasesが aws_cdk.core.CfnResource の時は、 cfn_options.condition = is_vpc """ ###################################################################### # create cmk of KMS to encrypt S3 bucket ###################################################################### kms_aes_siem = aws_kms.Key( self, 'KmsAesSiemLog', description='CMK for SIEM solution', removal_policy=core.RemovalPolicy.RETAIN) aws_kms.Alias( self, 'KmsAesSiemLogAlias', alias_name=kms_cmk_alias, target_key=kms_aes_siem, removal_policy=core.RemovalPolicy.RETAIN) kms_aes_siem.add_to_resource_policy( aws_iam.PolicyStatement( sid='Allow GuardDuty to use the key', actions=['kms:GenerateDataKey'], principals=[aws_iam.ServicePrincipal( 'guardduty.amazonaws.com')], resources=['*'],),) kms_aes_siem.add_to_resource_policy( aws_iam.PolicyStatement( sid='Allow VPC Flow Logs to use the key', actions=['kms:Encrypt', 'kms:Decrypt', 'kms:ReEncrypt*', 'kms:GenerateDataKey*', 'kms:DescribeKey'], principals=[aws_iam.ServicePrincipal( 'delivery.logs.amazonaws.com')], resources=['*'],),) # basic policy key_policy_basic1 = aws_iam.PolicyStatement( sid='Allow principals in the account to decrypt log files', actions=['kms:DescribeKey', 'kms:ReEncryptFrom'], principals=[aws_iam.AccountPrincipal( account_id=core.Aws.ACCOUNT_ID)], resources=['*'],) kms_aes_siem.add_to_resource_policy(key_policy_basic1) # for Athena key_policy_athena = aws_iam.PolicyStatement( sid='Allow Athena to query s3 objects with this key', actions=['kms:Decrypt', 'kms:DescribeKey', 'kms:Encrypt', 'kms:GenerateDataKey*', 'kms:ReEncrypt*'], principals=[aws_iam.AccountPrincipal( account_id=core.Aws.ACCOUNT_ID)], resources=['*'], conditions={'ForAnyValue:StringEquals': { 'aws:CalledVia': 'athena.amazonaws.com'}}) kms_aes_siem.add_to_resource_policy(key_policy_athena) # for CloudTrail key_policy_trail1 = aws_iam.PolicyStatement( sid='Allow CloudTrail to describe key', actions=['kms:DescribeKey'], principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], resources=['*'],) kms_aes_siem.add_to_resource_policy(key_policy_trail1) key_policy_trail2 = aws_iam.PolicyStatement( sid=('Allow CloudTrail to encrypt logs'), actions=['kms:GenerateDataKey*'], principals=[aws_iam.ServicePrincipal( 'cloudtrail.amazonaws.com')], resources=['*'], conditions={'StringLike': { 'kms:EncryptionContext:aws:cloudtrail:arn': [ f'arn:aws:cloudtrail:*:{core.Aws.ACCOUNT_ID}:trail/*']}}) kms_aes_siem.add_to_resource_policy(key_policy_trail2) ###################################################################### # create s3 bucket ###################################################################### block_pub = aws_s3.BlockPublicAccess( block_public_acls=True, ignore_public_acls=True, block_public_policy=True, restrict_public_buckets=True ) s3_geo = aws_s3.Bucket( self, 'S3BucketForGeoip', block_public_access=block_pub, bucket_name=s3bucket_name_geo, # removal_policy=core.RemovalPolicy.DESTROY, ) # create s3 bucket for log collector s3_log = aws_s3.Bucket( self, 'S3BucketForLog', block_public_access=block_pub, bucket_name=s3bucket_name_log, versioned=True, encryption=aws_s3.BucketEncryption.S3_MANAGED, # removal_policy=core.RemovalPolicy.DESTROY, ) # create s3 bucket for aes snapshot s3_snapshot = aws_s3.Bucket( self, 'S3BucketForSnapshot', block_public_access=block_pub, bucket_name=s3bucket_name_snapshot, # removal_policy=core.RemovalPolicy.DESTROY, ) ###################################################################### # IAM Role ###################################################################### # delopyment policy for lambda deploy-aes arn_prefix = f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}' loggroup_aes = f'log-group:/aws/aes/domains/{aes_domain_name}/*' loggroup_opensearch = ( f'log-group:/aws/OpenSearchService/domains/{aes_domain_name}/*') loggroup_lambda = 'log-group:/aws/lambda/aes-siem-*' policydoc_create_loggroup = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=[ 'logs:PutResourcePolicy', 'logs:DescribeLogGroups', 'logs:DescribeLogStreams' ], resources=[f'{arn_prefix}:*', ] ), aws_iam.PolicyStatement( actions=[ 'logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents', 'logs:PutRetentionPolicy'], resources=[ f'{arn_prefix}:{loggroup_aes}', f'{arn_prefix}:{loggroup_opensearch}', f'{arn_prefix}:{loggroup_lambda}', ], ) ] ) policydoc_crhelper = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=[ 'lambda:AddPermission', 'lambda:RemovePermission', 'events:ListRules', 'events:PutRule', 'events:DeleteRule', 'events:PutTargets', 'events:RemoveTargets'], resources=['*'] ) ] ) # snaphot rule for AES policydoc_snapshot = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=['s3:ListBucket'], resources=[s3_snapshot.bucket_arn] ), aws_iam.PolicyStatement( actions=['s3:GetObject', 's3:PutObject', 's3:DeleteObject'], resources=[s3_snapshot.bucket_arn + '/*'] ) ] ) aes_siem_snapshot_role = aws_iam.Role( self, 'AesSiemSnapshotRole', role_name='aes-siem-snapshot-role', inline_policies=[policydoc_snapshot, ], assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com') ) policydoc_assume_snapshrole = aws_iam.PolicyDocument( statements=[ aws_iam.PolicyStatement( actions=['iam:PassRole'], resources=[aes_siem_snapshot_role.role_arn] ), ] ) aes_siem_deploy_role_for_lambda = aws_iam.Role( self, 'AesSiemDeployRoleForLambda', role_name='aes-siem-deploy-role-for-lambda', managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonOpenSearchServiceFullAccess'), aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole'), ], inline_policies=[policydoc_assume_snapshrole, policydoc_snapshot, policydoc_create_loggroup, policydoc_crhelper], assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com') ) if vpc_type: aes_siem_deploy_role_for_lambda.add_managed_policy( aws_iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaVPCAccessExecutionRole') ) # for alert from Amazon OpenSearch Service aes_siem_sns_role = aws_iam.Role( self, 'AesSiemSnsRole', role_name='aes-siem-sns-role', assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com') ) # EC2 role aes_siem_es_loader_ec2_role = aws_iam.Role( self, 'AesSiemEsLoaderEC2Role', role_name='aes-siem-es-loader-for-ec2', assumed_by=aws_iam.ServicePrincipal('ec2.amazonaws.com'), ) aws_iam.CfnInstanceProfile( self, 'AesSiemEsLoaderEC2InstanceProfile', instance_profile_name=aes_siem_es_loader_ec2_role.role_name, roles=[aes_siem_es_loader_ec2_role.role_name] ) ###################################################################### # in VPC ###################################################################### aes_role_exist = check_iam_role('/aws-service-role/es.amazonaws.com/') if vpc_type and not aes_role_exist: slr_aes = aws_iam.CfnServiceLinkedRole( self, 'AWSServiceRoleForAmazonOpenSearchService', aws_service_name='es.amazonaws.com', description='Created by cloudformation of siem stack' ) slr_aes.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN ###################################################################### # SQS for es-laoder's DLQ ###################################################################### sqs_aes_siem_dlq = aws_sqs.Queue( self, 'AesSiemDlq', queue_name='aes-siem-dlq', retention_period=core.Duration.days(14)) sqs_aes_siem_splitted_logs = aws_sqs.Queue( self, 'AesSiemSqsSplitLogs', queue_name='aes-siem-sqs-splitted-logs', dead_letter_queue=aws_sqs.DeadLetterQueue( max_receive_count=2, queue=sqs_aes_siem_dlq), visibility_timeout=core.Duration.seconds(ES_LOADER_TIMEOUT), retention_period=core.Duration.days(14)) ###################################################################### # Setup Lambda ###################################################################### # setup lambda of es_loader lambda_es_loader_vpc_kwargs = {} if vpc_type: lambda_es_loader_vpc_kwargs = { 'security_group': sg_vpc_noinbound_aes_siem, 'vpc': vpc_aes_siem, 'vpc_subnets': vpc_subnets, } lambda_es_loader = aws_lambda.Function( self, 'LambdaEsLoader', **lambda_es_loader_vpc_kwargs, function_name='aes-siem-es-loader', description=f'{SOLUTION_NAME} / es-loader', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), # code=aws_lambda.Code.asset('../lambda/es_loader.zip'), code=aws_lambda.Code.asset('../lambda/es_loader'), handler='index.lambda_handler', memory_size=2048, timeout=core.Duration.seconds(ES_LOADER_TIMEOUT), reserved_concurrent_executions=( reserved_concurrency.value_as_number), dead_letter_queue_enabled=True, dead_letter_queue=sqs_aes_siem_dlq, environment={ 'GEOIP_BUCKET': s3bucket_name_geo, 'LOG_LEVEL': 'info', 'POWERTOOLS_LOGGER_LOG_EVENT': 'false', 'POWERTOOLS_SERVICE_NAME': 'es-loader', 'POWERTOOLS_METRICS_NAMESPACE': 'SIEM'}) es_loader_newver = lambda_es_loader.add_version( name=__version__, description=__version__) es_loader_opt = es_loader_newver.node.default_child.cfn_options es_loader_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN # send only # sqs_aes_siem_dlq.grant(lambda_es_loader, 'sqs:SendMessage') # send and reieve. but it must be loop sqs_aes_siem_dlq.grant( lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage', 'sqs:DeleteMessage', 'sqs:GetQueueAttributes') sqs_aes_siem_splitted_logs.grant( lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage', 'sqs:DeleteMessage', 'sqs:GetQueueAttributes') lambda_es_loader.add_event_source( aws_lambda_event_sources.SqsEventSource( sqs_aes_siem_splitted_logs, batch_size=1)) # es-loaer on EC2 role sqs_aes_siem_dlq.grant( aes_siem_es_loader_ec2_role, 'sqs:GetQueue*', 'sqs:ListQueues*', 'sqs:ReceiveMessage*', 'sqs:DeleteMessage*') lambda_geo = aws_lambda.Function( self, 'LambdaGeoipDownloader', function_name='aes-siem-geoip-downloader', description=f'{SOLUTION_NAME} / geoip-downloader', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), code=aws_lambda.Code.asset('../lambda/geoip_downloader'), handler='index.lambda_handler', memory_size=320, timeout=core.Duration.seconds(300), environment={ 's3bucket_name': s3bucket_name_geo, 'license_key': geoip_license_key.value_as_string, } ) lambda_geo_newver = lambda_geo.add_version( name=__version__, description=__version__) lamba_geo_opt = lambda_geo_newver.node.default_child.cfn_options lamba_geo_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN ###################################################################### # setup OpenSearch Service ###################################################################### lambda_deploy_es = aws_lambda.Function( self, 'LambdaDeployAES', function_name='aes-siem-deploy-aes', description=f'{SOLUTION_NAME} / opensearch domain deployment', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), # code=aws_lambda.Code.asset('../lambda/deploy_es.zip'), code=aws_lambda.Code.asset('../lambda/deploy_es'), handler='index.aes_domain_handler', memory_size=128, timeout=core.Duration.seconds(300), environment={ 'accountid': core.Aws.ACCOUNT_ID, 'aes_domain_name': aes_domain_name, 'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn, 'es_loader_role': lambda_es_loader.role.role_arn, 'allow_source_address': allow_source_address.value_as_string, }, role=aes_siem_deploy_role_for_lambda, ) lambda_deploy_es.add_environment( 's3_snapshot', s3_snapshot.bucket_name) if vpc_type: lambda_deploy_es.add_environment( 'vpc_subnet_id', subnet1.subnet_id) lambda_deploy_es.add_environment( 'security_group_id', sg_vpc_aes_siem.security_group_id) else: lambda_deploy_es.add_environment('vpc_subnet_id', 'None') lambda_deploy_es.add_environment('security_group_id', 'None') deploy_es_newver = lambda_deploy_es.add_version( name=__version__, description=__version__) deploy_es_opt = deploy_es_newver.node.default_child.cfn_options deploy_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN # execute lambda_deploy_es to deploy Amaozon ES Domain aes_domain = aws_cloudformation.CfnCustomResource( self, 'AesSiemDomainDeployedR2', service_token=lambda_deploy_es.function_arn,) aes_domain.add_override('Properties.ConfigVersion', __version__) es_endpoint = aes_domain.get_att('es_endpoint').to_string() lambda_es_loader.add_environment('ES_ENDPOINT', es_endpoint) lambda_es_loader.add_environment( 'SQS_SPLITTED_LOGS_URL', sqs_aes_siem_splitted_logs.queue_url) lambda_configure_es_vpc_kwargs = {} if vpc_type: lambda_configure_es_vpc_kwargs = { 'security_group': sg_vpc_noinbound_aes_siem, 'vpc': vpc_aes_siem, 'vpc_subnets': aws_ec2.SubnetSelection(subnets=[subnet1, ]), } lambda_configure_es = aws_lambda.Function( self, 'LambdaConfigureAES', **lambda_configure_es_vpc_kwargs, function_name='aes-siem-configure-aes', description=f'{SOLUTION_NAME} / opensearch configuration', runtime=aws_lambda.Runtime.PYTHON_3_8, architecture=aws_lambda.Architecture.X86_64, # architecture=region_mapping.find_in_map( # core.Aws.REGION, 'LambdaArm'), code=aws_lambda.Code.asset('../lambda/deploy_es'), handler='index.aes_config_handler', memory_size=128, timeout=core.Duration.seconds(300), environment={ 'accountid': core.Aws.ACCOUNT_ID, 'aes_domain_name': aes_domain_name, 'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn, 'es_loader_role': lambda_es_loader.role.role_arn, 'allow_source_address': allow_source_address.value_as_string, 'es_endpoint': es_endpoint, }, role=aes_siem_deploy_role_for_lambda, ) lambda_configure_es.add_environment( 's3_snapshot', s3_snapshot.bucket_name) if vpc_type: lambda_configure_es.add_environment( 'vpc_subnet_id', subnet1.subnet_id) lambda_configure_es.add_environment( 'security_group_id', sg_vpc_aes_siem.security_group_id) else: lambda_configure_es.add_environment('vpc_subnet_id', 'None') lambda_configure_es.add_environment('security_group_id', 'None') configure_es_newver = lambda_configure_es.add_version( name=__version__, description=__version__) configure_es_opt = configure_es_newver.node.default_child.cfn_options configure_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN aes_config = aws_cloudformation.CfnCustomResource( self, 'AesSiemDomainConfiguredR2', service_token=lambda_configure_es.function_arn,) aes_config.add_override('Properties.ConfigVersion', __version__) aes_config.add_depends_on(aes_domain) aes_config.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN es_arn = (f'arn:aws:es:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}' f':domain/{aes_domain_name}') # grant permission to es_loader role inline_policy_to_load_entries_into_es = aws_iam.Policy( self, 'aes-siem-policy-to-load-entries-to-es', policy_name='aes-siem-policy-to-load-entries-to-es', statements=[ aws_iam.PolicyStatement( actions=['es:*'], resources=[es_arn + '/*', ]), ] ) lambda_es_loader.role.attach_inline_policy( inline_policy_to_load_entries_into_es) aes_siem_es_loader_ec2_role.attach_inline_policy( inline_policy_to_load_entries_into_es) # grant additional permission to es_loader role additional_kms_cmks = self.node.try_get_context('additional_kms_cmks') if additional_kms_cmks: inline_policy_access_to_additional_cmks = aws_iam.Policy( self, 'access_to_additional_cmks', policy_name='access_to_additional_cmks', statements=[ aws_iam.PolicyStatement( actions=['kms:Decrypt'], resources=sorted(set(additional_kms_cmks)) ) ] ) lambda_es_loader.role.attach_inline_policy( inline_policy_access_to_additional_cmks) aes_siem_es_loader_ec2_role.attach_inline_policy( inline_policy_access_to_additional_cmks) additional_buckets = self.node.try_get_context('additional_s3_buckets') if additional_buckets: buckets_list = [] for bucket in additional_buckets: buckets_list.append(f'arn:aws:s3:::{bucket}') buckets_list.append(f'arn:aws:s3:::{bucket}/*') inline_policy_access_to_additional_buckets = aws_iam.Policy( self, 'access_to_additional_buckets', policy_name='access_to_additional_buckets', statements=[ aws_iam.PolicyStatement( actions=['s3:GetObject*', 's3:GetBucket*', 's3:List*'], resources=sorted(set(buckets_list)) ) ] ) lambda_es_loader.role.attach_inline_policy( inline_policy_access_to_additional_buckets) aes_siem_es_loader_ec2_role.attach_inline_policy( inline_policy_access_to_additional_buckets) kms_aes_siem.grant_decrypt(lambda_es_loader) kms_aes_siem.grant_decrypt(aes_siem_es_loader_ec2_role) ###################################################################### # s3 notification and grant permisssion ###################################################################### s3_geo.grant_read_write(lambda_geo) s3_geo.grant_read(lambda_es_loader) s3_geo.grant_read(aes_siem_es_loader_ec2_role) s3_log.grant_read(lambda_es_loader) s3_log.grant_read(aes_siem_es_loader_ec2_role) # create s3 notification for es_loader notification = aws_s3_notifications.LambdaDestination(lambda_es_loader) # assign notification for the s3 PUT event type # most log system use PUT, but also CLB use POST & Multipart Upload s3_log.add_event_notification( aws_s3.EventType.OBJECT_CREATED, notification, aws_s3.NotificationKeyFilter(prefix='AWSLogs/')) # For user logs, not AWS logs s3_log.add_event_notification( aws_s3.EventType.OBJECT_CREATED, notification, aws_s3.NotificationKeyFilter(prefix='UserLogs/')) # Download geoip to S3 once by executing lambda_geo get_geodb = aws_cloudformation.CfnCustomResource( self, 'ExecLambdaGeoipDownloader', service_token=lambda_geo.function_arn,) get_geodb.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN # Download geoip every day at 6PM UTC rule = aws_events.Rule( self, 'CwlRuleLambdaGeoipDownloaderDilly', schedule=aws_events.Schedule.rate(core.Duration.hours(12))) rule.add_target(aws_events_targets.LambdaFunction(lambda_geo)) ###################################################################### # bucket policy ###################################################################### s3_awspath = s3_log.bucket_arn + '/AWSLogs/' + core.Aws.ACCOUNT_ID bucket_policy_common1 = aws_iam.PolicyStatement( sid='ELB Policy', principals=[aws_iam.AccountPrincipal( account_id=region_mapping.find_in_map( core.Aws.REGION, 'ElbV2AccountId'))], actions=['s3:PutObject'], resources=[s3_awspath + '/*'],) # NLB / ALB / R53resolver / VPC Flow Logs bucket_policy_elb1 = aws_iam.PolicyStatement( sid='AWSLogDeliveryAclCheck For ALB NLB R53Resolver Flowlogs', principals=[aws_iam.ServicePrincipal( 'delivery.logs.amazonaws.com')], actions=['s3:GetBucketAcl', 's3:ListBucket'], resources=[s3_log.bucket_arn],) bucket_policy_elb2 = aws_iam.PolicyStatement( sid='AWSLogDeliveryWrite For ALB NLB R53Resolver Flowlogs', principals=[aws_iam.ServicePrincipal( 'delivery.logs.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_awspath + '/*'], conditions={ 'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_common1) s3_log.add_to_resource_policy(bucket_policy_elb1) s3_log.add_to_resource_policy(bucket_policy_elb2) # CloudTrail bucket_policy_trail1 = aws_iam.PolicyStatement( sid='AWSLogDeliveryAclCheck For Cloudtrail', principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], actions=['s3:GetBucketAcl'], resources=[s3_log.bucket_arn],) bucket_policy_trail2 = aws_iam.PolicyStatement( sid='AWSLogDeliveryWrite For CloudTrail', principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_awspath + '/*'], conditions={ 'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_trail1) s3_log.add_to_resource_policy(bucket_policy_trail2) # GuardDuty bucket_policy_gd1 = aws_iam.PolicyStatement( sid='Allow GuardDuty to use the getBucketLocation operation', principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')], actions=['s3:GetBucketLocation'], resources=[s3_log.bucket_arn],) bucket_policy_gd2 = aws_iam.PolicyStatement( sid='Allow GuardDuty to upload objects to the bucket', principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_log.bucket_arn + '/*'],) bucket_policy_gd5 = aws_iam.PolicyStatement( sid='Deny non-HTTPS access', effect=aws_iam.Effect.DENY, actions=['s3:*'], resources=[s3_log.bucket_arn + '/*'], conditions={'Bool': {'aws:SecureTransport': 'false'}}) bucket_policy_gd5.add_any_principal() s3_log.add_to_resource_policy(bucket_policy_gd1) s3_log.add_to_resource_policy(bucket_policy_gd2) s3_log.add_to_resource_policy(bucket_policy_gd5) # Config bucket_policy_config1 = aws_iam.PolicyStatement( sid='AWSConfig BucketPermissionsCheck and BucketExistenceCheck', principals=[aws_iam.ServicePrincipal('config.amazonaws.com')], actions=['s3:GetBucketAcl', 's3:ListBucket'], resources=[s3_log.bucket_arn],) bucket_policy_config2 = aws_iam.PolicyStatement( sid='AWSConfigBucketDelivery', principals=[aws_iam.ServicePrincipal('config.amazonaws.com')], actions=['s3:PutObject'], resources=[s3_awspath + '/Config/*'], conditions={ 'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_config1) s3_log.add_to_resource_policy(bucket_policy_config2) # geoip bucket_policy_geo1 = aws_iam.PolicyStatement( sid='Allow geoip downloader and es-loader to read/write', principals=[lambda_es_loader.role, lambda_geo.role], actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'], resources=[s3_geo.bucket_arn + '/*'],) s3_geo.add_to_resource_policy(bucket_policy_geo1) # ES Snapshot bucket_policy_snapshot = aws_iam.PolicyStatement( sid='Allow ES to store snapshot', principals=[aes_siem_snapshot_role], actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'], resources=[s3_snapshot.bucket_arn + '/*'],) s3_snapshot.add_to_resource_policy(bucket_policy_snapshot) ###################################################################### # for multiaccount / organizaitons ###################################################################### if org_id or no_org_ids: ################################################################## # KMS key policy for multiaccount / organizaitons ################################################################## # for CloudTrail cond_tail2 = self.make_resource_list( path='arn:aws:cloudtrail:*:', tail=':trail/*', keys=self.list_without_none(org_mgmt_id, no_org_ids)) key_policy_mul_trail2 = aws_iam.PolicyStatement( sid=('Allow CloudTrail to encrypt logs for multiaccounts'), actions=['kms:GenerateDataKey*'], principals=[aws_iam.ServicePrincipal( 'cloudtrail.amazonaws.com')], resources=['*'], conditions={'StringLike': { 'kms:EncryptionContext:aws:cloudtrail:arn': cond_tail2}}) kms_aes_siem.add_to_resource_policy(key_policy_mul_trail2) # for replicaiton key_policy_rep1 = aws_iam.PolicyStatement( sid=('Enable cross account encrypt access for S3 Cross Region ' 'Replication'), actions=['kms:Encrypt'], principals=self.make_account_principals( org_mgmt_id, org_member_ids, no_org_ids), resources=['*'],) kms_aes_siem.add_to_resource_policy(key_policy_rep1) ################################################################## # Buckdet Policy for multiaccount / organizaitons ################################################################## s3_log_bucket_arn = 'arn:aws:s3:::' + s3bucket_name_log # for CloudTrail s3_mulpaths = self.make_resource_list( path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/*', keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids)) bucket_policy_org_trail = aws_iam.PolicyStatement( sid='AWSCloudTrailWrite for Multiaccounts / Organizations', principals=[ aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')], actions=['s3:PutObject'], resources=s3_mulpaths, conditions={'StringEquals': { 's3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_org_trail) # config s3_conf_multpaths = self.make_resource_list( path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/Config/*', keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids)) bucket_policy_mul_config2 = aws_iam.PolicyStatement( sid='AWSConfigBucketDelivery', principals=[aws_iam.ServicePrincipal('config.amazonaws.com')], actions=['s3:PutObject'], resources=s3_conf_multpaths, conditions={'StringEquals': { 's3:x-amz-acl': 'bucket-owner-full-control'}}) s3_log.add_to_resource_policy(bucket_policy_mul_config2) # for replication bucket_policy_rep1 = aws_iam.PolicyStatement( sid='PolicyForDestinationBucket / Permissions on objects', principals=self.make_account_principals( org_mgmt_id, org_member_ids, no_org_ids), actions=['s3:ReplicateDelete', 's3:ReplicateObject', 's3:ReplicateTags', 's3:GetObjectVersionTagging', 's3:ObjectOwnerOverrideToBucketOwner'], resources=[f'{s3_log_bucket_arn}/*']) bucket_policy_rep2 = aws_iam.PolicyStatement( sid='PolicyForDestinationBucket / Permissions on bucket', principals=self.make_account_principals( org_mgmt_id, org_member_ids, no_org_ids), actions=['s3:List*', 's3:GetBucketVersioning', 's3:PutBucketVersioning'], resources=[f'{s3_log_bucket_arn}']) s3_log.add_to_resource_policy(bucket_policy_rep1) s3_log.add_to_resource_policy(bucket_policy_rep2) ###################################################################### # SNS topic for Amazon OpenSearch Service Alert ###################################################################### sns_topic = aws_sns.Topic( self, 'SnsTopic', topic_name='aes-siem-alert', display_name='AES SIEM') sns_topic.add_subscription(aws_sns_subscriptions.EmailSubscription( email_address=sns_email.value_as_string)) sns_topic.grant_publish(aes_siem_sns_role) ###################################################################### # output of CFn ###################################################################### kibanaurl = f'https://{es_endpoint}/_dashboards/' kibanaadmin = aes_domain.get_att('kibanaadmin').to_string() kibanapass = aes_domain.get_att('kibanapass').to_string() core.CfnOutput(self, 'RoleDeploy', export_name='role-deploy', value=aes_siem_deploy_role_for_lambda.role_arn) core.CfnOutput(self, 'DashboardsUrl', export_name='dashboards-url', value=kibanaurl) core.CfnOutput(self, 'DashboardsPassword', export_name='dashboards-pass', value=kibanapass, description=('Please change the password in OpenSearch ' 'Dashboards ASAP')) core.CfnOutput(self, 'DashboardsAdminID', export_name='dashboards-admin', value=kibanaadmin)
def __init__(self, scope: core.Construct, id: str, custom: dict, **kwargs) -> None: super().__init__(scope, id, **kwargs) region = self.region #create S3 access role for ec2 ec2Role = iam.Role( self, "aws-cdk-handson-lab02-ec2role", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3ReadOnlyAccess") ]) instanceProfile = iam.CfnInstanceProfile( self, "aws-cdk-handson-lab02-ec2Profile", roles=[ec2Role.role_name], instance_profile_name="aws-cdk-handson-lab02-ec2Profile", ) #create new VPC for lab02 #vpc = ec2.Vpc.from_lookup(self, "aws-cdk-handson-lab02-vpc",vpc_name="default") #使用默认的vpc #创建新的vpc vpc = ec2.Vpc(self, id="aws-cdk-handson-lab02-vpc", cidr="172.30.0.0/16", nat_gateways=0, subnet_configuration=[ { "cidrMask": 24, "name": "subnet-1-", "subnetType": ec2.SubnetType.PUBLIC }, { "cidrMask": 24, "name": "subnet-2-", "subnetType": ec2.SubnetType.PUBLIC }, { "cidrMask": 24, "name": "subnet-3-", "subnetType": ec2.SubnetType.PUBLIC }, ]) #使用已有的安全组 #sg=ec2.SecurityGroup.from_security_group_id(self,"nodeSG",security_group_id='sg-0dd53aaa5c9eb8324') #创建新的安全组 sg = ec2.CfnSecurityGroup( self, "aws-cdk-handson-lab02-ec2securitygroup", group_description="this is aws-cdk-handson workshop", group_name="aws-cdk-handson-lab02-ec2securitygroup", security_group_ingress=[ { "ipProtocol": "tcp", "fromPort": 80, "toPort": 80, "cidrIp": "0.0.0.0/0", }, { "ipProtocol": "tcp", "fromPort": 22, "toPort": 22, "cidrIp": "0.0.0.0/0", }, { "ipProtocol": "tcp", "fromPort": 8080, "toPort": 8080, "cidrIp": "0.0.0.0/0", }, ], vpc_id=vpc.vpc_id) #read and base64 encode userdata file data = open("../resource/httpd.sh", "rb").read() encodedBytes = base64.encodebytes(data) encodedStr = str(encodedBytes, "utf-8") #create ec2 instances ami = ec2.AmazonLinuxImage( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2) #创建2台EC2 ec2Count = 2 for i in range(ec2Count): eni0 = ec2.CfnNetworkInterface( self, "eni-" + str(i), subnet_id=vpc.public_subnets[0].subnet_id, group_set=[sg.attr_group_id]) #group_set=[sg.security_group_id] instance = ec2.CfnInstance( self, "ec2-httpd-" + str(i), image_id=ami.get_image( self).image_id, #use Amazon Linux 2 AMI instance_type="t3.micro", key_name="wsu-cn-northwest-1", #这个是keypair的名字非常重要 tags=[ core.CfnTag(key="Name", value="aws-cdk-lab02-ec2-" + str(i)) ], #加上标签 iam_instance_profile=instanceProfile.ref, user_data=encodedStr, network_interfaces=[{ 'deviceIndex': '0', 'networkInterfaceId': eni0.ref, }]) core.CfnOutput(self, "PublicIP-" + str(i), export_name="PublicIP-" + str(i), value=instance.attr_public_ip) core.CfnOutput(self, "PublicDNSName-" + str(i), export_name="PublicDNSName-" + str(i), value=instance.attr_public_dns_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) eks_vpc = ec2.Vpc(self, "VPC", cidr="10.0.0.0/16") self.eks_vpc = eks_vpc # Create IAM Role For code-server bastion bastion_role = iam.Role( self, "BastionRole", assumed_by=iam.CompositePrincipal( iam.ServicePrincipal("ec2.amazonaws.com"), iam.AccountRootPrincipal()), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( "AdministratorAccess") ]) self.bastion_role = bastion_role # Create EC2 Instance Profile for that Role instance_profile = iam.CfnInstanceProfile( self, "InstanceProfile", roles=[bastion_role.role_name]) # Create SecurityGroup for the Control Plane ENIs eks_security_group = ec2.SecurityGroup(self, "EKSSecurityGroup", vpc=eks_vpc, allow_all_outbound=True) eks_security_group.add_ingress_rule(ec2.Peer.ipv4('10.0.0.0/16'), ec2.Port.all_traffic()) # Create an EKS Cluster eks_cluster = eks.Cluster( self, "cluster", cluster_name="cluster", vpc=eks_vpc, masters_role=bastion_role, default_capacity_type=eks.DefaultCapacityType.NODEGROUP, default_capacity_instance=ec2.InstanceType("m5.large"), default_capacity=2, security_group=eks_security_group, endpoint_access=eks.EndpointAccess.PUBLIC_AND_PRIVATE, version=eks.KubernetesVersion.V1_17) self.cluster_cert = eks_cluster.cluster_certificate_authority_data # Deploy ALB Ingress Controller # Create the k8s Service account and corresponding IAM Role mapped via IRSA alb_service_account = eks_cluster.add_service_account( "alb-ingress-controller", name="alb-ingress-controller", namespace="kube-system") # Create the PolicyStatements to attach to the role # I couldn't find a way to get this to work with a PolicyDocument and there are 10 of these alb_policy_statement_json_1 = { "Effect": "Allow", "Action": [ "acm:DescribeCertificate", "acm:ListCertificates", "acm:GetCertificate" ], "Resource": "*" } alb_policy_statement_json_2 = { "Effect": "Allow", "Action": [ "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateSecurityGroup", "ec2:CreateTags", "ec2:DeleteTags", "ec2:DeleteSecurityGroup", "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "ec2:DescribeInternetGateways", "ec2:DescribeNetworkInterfaces", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVpcs", "ec2:ModifyInstanceAttribute", "ec2:ModifyNetworkInterfaceAttribute", "ec2:RevokeSecurityGroupIngress" ], "Resource": "*" } alb_policy_statement_json_3 = { "Effect": "Allow", "Action": [ "elasticloadbalancing:AddListenerCertificates", "elasticloadbalancing:AddTags", "elasticloadbalancing:CreateListener", "elasticloadbalancing:CreateLoadBalancer", "elasticloadbalancing:CreateRule", "elasticloadbalancing:CreateTargetGroup", "elasticloadbalancing:DeleteListener", "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:DeleteRule", "elasticloadbalancing:DeleteTargetGroup", "elasticloadbalancing:DeregisterTargets", "elasticloadbalancing:DescribeListenerCertificates", "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeRules", "elasticloadbalancing:DescribeSSLPolicies", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:ModifyListener", "elasticloadbalancing:ModifyLoadBalancerAttributes", "elasticloadbalancing:ModifyRule", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:RegisterTargets", "elasticloadbalancing:RemoveListenerCertificates", "elasticloadbalancing:RemoveTags", "elasticloadbalancing:SetIpAddressType", "elasticloadbalancing:SetSecurityGroups", "elasticloadbalancing:SetSubnets", "elasticloadbalancing:SetWebAcl" ], "Resource": "*" } alb_policy_statement_json_4 = { "Effect": "Allow", "Action": [ "iam:CreateServiceLinkedRole", "iam:GetServerCertificate", "iam:ListServerCertificates" ], "Resource": "*" } alb_policy_statement_json_5 = { "Effect": "Allow", "Action": ["cognito-idp:DescribeUserPoolClient"], "Resource": "*" } alb_policy_statement_json_6 = { "Effect": "Allow", "Action": [ "waf-regional:GetWebACLForResource", "waf-regional:GetWebACL", "waf-regional:AssociateWebACL", "waf-regional:DisassociateWebACL" ], "Resource": "*" } alb_policy_statement_json_7 = { "Effect": "Allow", "Action": ["tag:GetResources", "tag:TagResources"], "Resource": "*" } alb_policy_statement_json_8 = { "Effect": "Allow", "Action": ["waf:GetWebACL"], "Resource": "*" } alb_policy_statement_json_9 = { "Effect": "Allow", "Action": [ "wafv2:GetWebACL", "wafv2:GetWebACLForResource", "wafv2:AssociateWebACL", "wafv2:DisassociateWebACL" ], "Resource": "*" } alb_policy_statement_json_10 = { "Effect": "Allow", "Action": [ "shield:DescribeProtection", "shield:GetSubscriptionState", "shield:DeleteProtection", "shield:CreateProtection", "shield:DescribeSubscription", "shield:ListProtections" ], "Resource": "*" } # Attach the necessary permissions alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_1)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_2)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_3)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_4)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_5)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_6)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_7)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_8)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_9)) alb_service_account.add_to_policy( iam.PolicyStatement.from_json(alb_policy_statement_json_10)) # Deploy the ALB Ingress Controller from the Helm chart eks_cluster.add_helm_chart( "aws-alb-ingress-controller", chart="aws-alb-ingress-controller", repository= "http://storage.googleapis.com/kubernetes-charts-incubator", namespace="kube-system", values={ "clusterName": "cluster", "awsRegion": os.environ["CDK_DEFAULT_REGION"], "awsVpcID": eks_vpc.vpc_id, "rbac": { "create": True, "serviceAccount": { "create": False, "name": "alb-ingress-controller" } } }) # Create code-server bastion # Get Latest Amazon Linux AMI amzn_linux = ec2.MachineImage.latest_amazon_linux( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2, edition=ec2.AmazonLinuxEdition.STANDARD, virtualization=ec2.AmazonLinuxVirt.HVM, storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE) # Create SecurityGroup for code-server security_group = ec2.SecurityGroup(self, "SecurityGroup", vpc=eks_vpc, allow_all_outbound=True) security_group.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(8080)) # Create our EC2 instance running CodeServer code_server_instance = ec2.Instance( self, "CodeServerInstance", instance_type=ec2.InstanceType("t3.large"), machine_image=amzn_linux, role=bastion_role, vpc=eks_vpc, vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), security_group=security_group, block_devices=[ ec2.BlockDevice(device_name="/dev/xvda", volume=ec2.BlockDeviceVolume.ebs(20)) ]) # Add UserData code_server_instance.user_data.add_commands( "mkdir -p ~/.local/lib ~/.local/bin ~/.config/code-server") code_server_instance.user_data.add_commands( "curl -fL https://github.com/cdr/code-server/releases/download/v3.5.0/code-server-3.5.0-linux-amd64.tar.gz | tar -C ~/.local/lib -xz" ) code_server_instance.user_data.add_commands( "mv ~/.local/lib/code-server-3.5.0-linux-amd64 ~/.local/lib/code-server-3.5.0" ) code_server_instance.user_data.add_commands( "ln -s ~/.local/lib/code-server-3.5.0/bin/code-server ~/.local/bin/code-server" ) code_server_instance.user_data.add_commands( "echo \"bind-addr: 0.0.0.0:8080\" > ~/.config/code-server/config.yaml" ) code_server_instance.user_data.add_commands( "echo \"auth: password\" >> ~/.config/code-server/config.yaml") code_server_instance.user_data.add_commands( "echo \"password: $(curl -s http://169.254.169.254/latest/meta-data/instance-id)\" >> ~/.config/code-server/config.yaml" ) code_server_instance.user_data.add_commands( "echo \"cert: false\" >> ~/.config/code-server/config.yaml") code_server_instance.user_data.add_commands( "~/.local/bin/code-server &") code_server_instance.user_data.add_commands( "yum -y install jq gettext bash-completion moreutils") code_server_instance.user_data.add_commands( "sudo pip install --upgrade awscli && hash -r") code_server_instance.user_data.add_commands( "echo 'export ALB_INGRESS_VERSION=\"v1.1.8\"' >> ~/.bash_profile") code_server_instance.user_data.add_commands( "curl --silent --location -o /usr/local/bin/kubectl \"https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.9/2020-08-04/bin/linux/amd64/kubectl\"" ) code_server_instance.user_data.add_commands( "chmod +x /usr/local/bin/kubectl") code_server_instance.user_data.add_commands( "curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash" ) code_server_instance.user_data.add_commands( "export ACCOUNT_ID=$(aws sts get-caller-identity --output text --query Account)" ) code_server_instance.user_data.add_commands( "export AWS_REGION=$(curl -s 169.254.169.254/latest/dynamic/instance-identity/document | jq -r '.region')" ) code_server_instance.user_data.add_commands( "echo \"export ACCOUNT_ID=${ACCOUNT_ID}\" | tee -a ~/.bash_profile" ) code_server_instance.user_data.add_commands( "echo \"export AWS_REGION=${AWS_REGION}\" | tee -a ~/.bash_profile" ) code_server_instance.user_data.add_commands( "aws configure set default.region ${AWS_REGION}") code_server_instance.user_data.add_commands( "curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -" ) code_server_instance.user_data.add_commands("yum -y install nodejs") code_server_instance.user_data.add_commands( "amazon-linux-extras enable python3") code_server_instance.user_data.add_commands( "yum install -y python3 --disablerepo amzn2-core") code_server_instance.user_data.add_commands("yum install -y git") code_server_instance.user_data.add_commands( "rm /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python && ln -s /usr/bin/pip3 /usr/bin/pip" ) code_server_instance.user_data.add_commands("npm install -g aws-cdk") code_server_instance.user_data.add_commands( "echo 'export KUBECONFIG=~/.kube/config' >> ~/.bash_profile") code_server_instance.user_data.add_commands( "git clone https://github.com/jasonumiker/eks-school.git") # Add ALB lb = elbv2.ApplicationLoadBalancer(self, "LB", vpc=eks_vpc, internet_facing=True) listener = lb.add_listener("Listener", port=80) listener.connections.allow_default_port_from_any_ipv4( "Open to the Internet") listener.connections.allow_to_any_ipv4( port_range=ec2.Port(string_representation="TCP 8080", protocol=ec2.Protocol.TCP, from_port=8080, to_port=8080)) listener.add_targets( "Target", port=8080, targets=[ elbv2.InstanceTarget( instance_id=code_server_instance.instance_id, port=8080) ])
def __init__(self, scope: core.Construct, id: str, landing_zone: ILandingZone, directory: DirectoryServicesConstruct, group_names: [List], **kwargs) -> None: super().__init__(scope, id, **kwargs) self.__landing_zone = landing_zone # Configure the security groups self.security_group = ec2.SecurityGroup( self, 'SecurityGroup', vpc=landing_zone.networking.vpc, allow_all_outbound=True, description='HadoopConstruct Security Group', security_group_name='hadoop-mapreduce-group') for port in services.keys(): self.security_group.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port(protocol=ec2.Protocol.TCP, from_port=port, to_port=port, string_representation=services[port])) self.security_group.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port(protocol=ec2.Protocol.UDP, from_port=0, to_port=65535, string_representation='Allow All UDP Traffic')) self.security_group.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port(protocol=ec2.Protocol.TCP, from_port=0, to_port=65535, string_representation='Allow All TCP Traffic')) # Setup roles... self.jobFlowRole = iam.Role( self, 'JobFlowRole', assumed_by=iam.ServicePrincipal(service='ec2.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore'), iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonElasticMapReduceforEC2Role'), ]) profile_name = 'jobflowprofile@{}-{}'.format( landing_zone.zone_name, core.Stack.of(self).region) job_flow_instance_profile = iam.CfnInstanceProfile( self, 'JobFlowInstanceProfile', instance_profile_name=profile_name, roles=[self.jobFlowRole.role_name]) serviceRole = iam.Role( self, 'ServiceRole', assumed_by=iam.ServicePrincipal( service='elasticmapreduce.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonElasticMapReduceRole') ]) self.database = g.Database(self, 'GlueStore', database_name='demo-database') self.bucket = s3.Bucket(self, 'LogBucket', removal_policy=core.RemovalPolicy.DESTROY) emr_fs = EmrfsConstruct(self, 'Emrfs', landing_zone=landing_zone, directory=directory, group_names=group_names, job_flow_role=self.jobFlowRole) # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticmapreduce-instancefleetconfig.html self.cluster = emr.CfnCluster( self, 'Hadoop', name='HadoopCluster', job_flow_role=profile_name, #'EMR_EC2_DefaultRole', service_role=serviceRole.role_name, log_uri='s3://' + self.bucket.bucket_name + '/logs', release_label='emr-6.2.0', applications=[ emr.CfnCluster.ApplicationProperty(name='Spark'), emr.CfnCluster.ApplicationProperty(name='Presto'), emr.CfnCluster.ApplicationProperty(name='Hue'), emr.CfnCluster.ApplicationProperty(name='Hive'), emr.CfnCluster.ApplicationProperty(name='JupyterHub'), ], configurations=[ emr.CfnCluster.ConfigurationProperty( classification='spark-hive-site', configuration_properties={ 'hive.metastore.client.factory.class': 'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory' }), emr.CfnCluster.ConfigurationProperty( classification='hive-site', configuration_properties={ 'hive.metastore.client.factory.class': 'com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory', 'aws.glue.partition.num.segments': '10', #1 to 10; (default=5) 'hive.metastore.schema.verification': 'false', }) ], security_configuration=emr_fs.security_configuration.ref, # kerberos_attributes= emr.CfnCluster.KerberosAttributesProperty( # kdc_admin_password=directory.password, # realm= directory.mad.name.upper(), # ad_domain_join_password=directory.password, # ad_domain_join_user= directory.admin # ), managed_scaling_policy=emr.CfnCluster.ManagedScalingPolicyProperty( compute_limits=emr.CfnCluster.ComputeLimitsProperty( minimum_capacity_units=1, maximum_capacity_units=25, unit_type='InstanceFleetUnits')), instances=emr.CfnCluster.JobFlowInstancesConfigProperty( #hadoop_version='2.4.0', termination_protected=False, master_instance_fleet=emr.CfnCluster. InstanceFleetConfigProperty( target_spot_capacity=1, instance_type_configs=[ emr.CfnCluster.InstanceTypeConfigProperty( instance_type='m5.xlarge', ) ]), core_instance_fleet=emr.CfnCluster.InstanceFleetConfigProperty( target_spot_capacity=1, instance_type_configs=[ emr.CfnCluster.InstanceTypeConfigProperty( instance_type='m5.xlarge', ebs_configuration=emr.CfnCluster. EbsConfigurationProperty(ebs_block_device_configs=[ emr.CfnCluster.EbsBlockDeviceConfigProperty( volume_specification=emr.CfnCluster. VolumeSpecificationProperty( size_in_gb=50, volume_type='gp2')) ])) ]), additional_master_security_groups=[ self.security_group.security_group_id ], additional_slave_security_groups=[ self.security_group.security_group_id ], ec2_subnet_ids=[ net.subnet_id for net in landing_zone.networking.vpc. _select_subnet_objects(subnet_group_name='Hadoop') ], )) self.cluster.add_depends_on(job_flow_instance_profile)
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None: super().__init__(scope, id, **kwargs) ################################################################################ # Set up permissions ro_buckets = set() for bucket in props['ro_buckets']: tmp_bucket = s3.Bucket.from_bucket_name(self, bucket, bucket_name=bucket) ro_buckets.add(tmp_bucket) rw_buckets = set() for bucket in props['rw_buckets']: tmp_bucket = s3.Bucket.from_bucket_name(self, bucket, bucket_name=bucket) rw_buckets.add(tmp_bucket) batch_service_role = iam.Role( self, 'BatchServiceRole', assumed_by=iam.ServicePrincipal('batch.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSBatchServiceRole') ]) spotfleet_role = iam.Role( self, 'AmazonEC2SpotFleetRole', assumed_by=iam.ServicePrincipal('spotfleet.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonEC2SpotFleetTaggingRole') ]) # Create role for Batch instances batch_instance_role = iam.Role( self, 'BatchInstanceRole', role_name='RnasumBatchInstanceRole', assumed_by=iam.CompositePrincipal( iam.ServicePrincipal('ec2.amazonaws.com'), iam.ServicePrincipal('ecs.amazonaws.com')), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonEC2RoleforSSM'), iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonEC2ContainerServiceforEC2Role') ]) batch_instance_role.add_to_policy( iam.PolicyStatement(actions=[ "ec2:Describe*", "ec2:AttachVolume", "ec2:CreateVolume", "ec2:CreateTags", "ec2:ModifyInstanceAttribute" ], resources=["*"])) batch_instance_role.add_to_policy( iam.PolicyStatement(actions=["ecs:ListClusters"], resources=["*"])) for bucket in ro_buckets: bucket.grant_read(batch_instance_role) for bucket in rw_buckets: # TODO: restirct write to paths with */rnasum/* bucket.grant_read_write(batch_instance_role) # Turn the instance role into a Instance Profile batch_instance_profile = iam.CfnInstanceProfile( self, 'BatchInstanceProfile', instance_profile_name='RnasumBatchInstanceProfile', roles=[batch_instance_role.role_name]) ################################################################################ # Minimal networking # TODO: import resource created with TF vpc = props['vpc'] ################################################################################ # Setup Batch compute resources # Configure BlockDevice to expand instance disk space (if needed?) block_device_mappings = [{ 'deviceName': '/dev/xvdf', 'ebs': { 'deleteOnTermination': True, 'volumeSize': 1024, 'volumeType': 'gp2' } }] launch_template = ec2.CfnLaunchTemplate( self, 'RnasumBatchComputeLaunchTemplate', launch_template_name='RnasumBatchComputeLaunchTemplate', launch_template_data={ # 'userData': core.Fn.base64(user_data_script), FIXME may not need this for RNAsum case? see job_definition below 'blockDeviceMappings': block_device_mappings }) launch_template_spec = batch.LaunchTemplateSpecification( launch_template_name=launch_template.launch_template_name, version='$Latest') my_compute_res = batch.ComputeResources( type=batch.ComputeResourceType.SPOT, allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE, desiredv_cpus=0, maxv_cpus=80, minv_cpus=0, image=ec2.MachineImage.generic_linux( ami_map={'ap-southeast-2': props['compute_env_ami']}), launch_template=launch_template_spec, spot_fleet_role=spotfleet_role, instance_role=batch_instance_profile.instance_profile_name, vpc=vpc, #compute_resources_tags=core.Tag('Creator', 'Batch') ) # XXX: How to add more than one tag above?? # core.Tag.add(my_compute_res, 'Foo', 'Bar') my_compute_env = batch.ComputeEnvironment( self, 'RnasumBatchComputeEnv', compute_environment_name="RnasumBatchComputeEnv", service_role=batch_service_role, compute_resources=my_compute_res) job_queue = batch.JobQueue(self, 'RnasumJobQueue', job_queue_name='rnasum_job_queue', compute_environments=[ batch.JobQueueComputeEnvironment( compute_environment=my_compute_env, order=1) ], priority=10) # it is equivalent of # https://github.com/umccr/infrastructure/blob/master/terraform/stacks/wts_report/jobs/wts_report.json default_container_props = { 'image': props['container_image'], 'vcpus': 2, 'memory': 2048, 'command': ['/opt/container/WTS-report-wrapper.sh', 'Ref::vcpus'], 'volumes': [{ 'host': { 'sourcePath': '/mnt' }, 'name': 'work' }, { 'host': { 'sourcePath': '/opt/container' }, 'name': 'container' }], 'mountPoints': [{ 'containerPath': '/work', 'readOnly': False, 'sourceVolume': 'work' }, { 'containerPath': '/opt/container', 'readOnly': True, 'sourceVolume': 'container' }], 'readonlyRootFilesystem': False, 'privileged': True, 'ulimits': [] } # and CDK equivalent of # https://github.com/umccr/infrastructure/blob/master/terraform/stacks/wts_report/main.tf#L113 job_definition = batch.CfnJobDefinition( self, 'RnasumJobDefinition', job_definition_name='rnasum_job_dev', type='container', container_properties=default_container_props, parameters={ 'vcpus': 1, }) ################################################################################ # Set up job submission Lambda lambda_role = iam.Role( self, 'RnasumLambdaRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole'), iam.ManagedPolicy.from_aws_managed_policy_name( 'AWSBatchFullAccess') # TODO: restrict! ]) for bucket in ro_buckets: bucket.grant_read(lambda_role) for bucket in rw_buckets: bucket.grant_read(lambda_role) # TODO: support dev/prod split, i.e. image being configurable on dev, but fixed on prod # may need a default JobDefinition to be set up # and CDK equivalent of # https://github.com/umccr/infrastructure/blob/master/terraform/stacks/wts_report/main.tf#L159 lmbda.Function(self, 'RnasumLambda', function_name='rnasum_batch_lambda', handler='trigger_wts_report.lambda_handler', runtime=lmbda.Runtime.PYTHON_3_7, code=lmbda.Code.from_asset('lambdas/'), environment={ 'JOBNAME_PREFIX': "rnasum_", 'JOBQUEUE': job_queue.job_queue_name, 'JOBDEF': job_definition.job_definition_name, 'REFDATA_BUCKET': props['refdata_bucket'], 'DATA_BUCKET': props['data_bucket'], 'JOB_MEM': '32000', 'JOB_VCPUS': '8', 'REF_DATASET': 'PANCAN', 'GENOME_BUILD': '38', }, role=lambda_role)
def __init__(self, scope: core.Construct, id: str, props, **kwargs) -> None: super().__init__(scope, id, **kwargs) ################################################################################ # Set up permissions ro_buckets = set() for bucket in props['ro_buckets']: tmp_bucket = s3.Bucket.from_bucket_name(self, bucket, bucket_name=bucket) ro_buckets.add(tmp_bucket) rw_buckets = set() for bucket in props['rw_buckets']: tmp_bucket = s3.Bucket.from_bucket_name(self, bucket, bucket_name=bucket) rw_buckets.add(tmp_bucket) batch_service_role = iam.Role( self, 'BatchServiceRole', assumed_by=iam.ServicePrincipal('batch.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSBatchServiceRole') ]) spotfleet_role = iam.Role( self, 'AmazonEC2SpotFleetRole', assumed_by=iam.ServicePrincipal('spotfleet.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonEC2SpotFleetTaggingRole') ]) # Create role for Batch instances batch_instance_role = iam.Role( self, 'BatchInstanceRole', role_name='UmccriseBatchInstanceRole', assumed_by=iam.CompositePrincipal( iam.ServicePrincipal('ec2.amazonaws.com'), iam.ServicePrincipal('ecs.amazonaws.com')), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonEC2RoleforSSM'), iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonEC2ContainerServiceforEC2Role') ]) batch_instance_role.add_to_policy( iam.PolicyStatement(actions=[ "ec2:Describe*", "ec2:AttachVolume", "ec2:CreateVolume", "ec2:CreateTags", "ec2:ModifyInstanceAttribute" ], resources=["*"])) batch_instance_role.add_to_policy( iam.PolicyStatement(actions=["ecs:ListClusters"], resources=["*"])) for bucket in ro_buckets: bucket.grant_read(batch_instance_role) for bucket in rw_buckets: # restirct write to paths with */umccrise/* bucket.grant_read_write(batch_instance_role, '*/umccrised/*') # Turn the instance role into a Instance Profile batch_instance_profile = iam.CfnInstanceProfile( self, 'BatchInstanceProfile', instance_profile_name='UmccriseBatchInstanceProfile', roles=[batch_instance_role.role_name]) ################################################################################ # Minimal networking # TODO: import resource created with TF vpc = props['vpc'] ################################################################################ # Setup Batch compute resources # Configure BlockDevice to expand instance disk space (if needed?) block_device_mappings = [{ 'deviceName': '/dev/xvdf', 'ebs': { 'deleteOnTermination': True, 'volumeSize': 1024, 'volumeType': 'gp2' } }] launch_template = ec2.CfnLaunchTemplate( self, 'UmccriseBatchComputeLaunchTemplate', launch_template_name='UmccriseBatchComputeLaunchTemplate', launch_template_data={ 'userData': core.Fn.base64(user_data_script), 'blockDeviceMappings': block_device_mappings }) launch_template_spec = batch.LaunchTemplateSpecification( launch_template_name=launch_template.launch_template_name, version='$Latest') my_compute_res = batch.ComputeResources( type=batch.ComputeResourceType.SPOT, allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE, desiredv_cpus=0, maxv_cpus=128, minv_cpus=0, image=ec2.MachineImage.generic_linux( ami_map={'ap-southeast-2': props['compute_env_ami']}), launch_template=launch_template_spec, spot_fleet_role=spotfleet_role, instance_role=batch_instance_profile.instance_profile_name, vpc=vpc, #compute_resources_tags=core.Tag('Creator', 'Batch') ) # XXX: How to add more than one tag above?? # core.Tag.add(my_compute_res, 'Foo', 'Bar') my_compute_env = batch.ComputeEnvironment( self, 'UmccriseBatchComputeEnv', compute_environment_name="cdk-umccrise-batch-compute-env", service_role=batch_service_role, compute_resources=my_compute_res) job_queue = batch.JobQueue(self, 'UmccriseJobQueue', job_queue_name='cdk-umccrise_job_queue', compute_environments=[ batch.JobQueueComputeEnvironment( compute_environment=my_compute_env, order=1) ], priority=10) job_container = batch.JobDefinitionContainer( image=ecs.ContainerImage.from_registry( name=props['container_image']), vcpus=2, memory_limit_mib=2048, command=["/opt/container/umccrise-wrapper.sh", "Ref::vcpus"], mount_points=[ ecs.MountPoint(container_path='/work', read_only=False, source_volume='work'), ecs.MountPoint(container_path='/opt/container', read_only=True, source_volume='container') ], volumes=[ ecs.Volume(name='container', host=ecs.Host(source_path='/opt/container')), ecs.Volume(name='work', host=ecs.Host(source_path='/mnt')) ], privileged=True) job_definition = batch.JobDefinition( self, 'UmccriseJobDefinition', job_definition_name='cdk-umccrise-job-definition', parameters={'vcpus': '1'}, container=job_container, timeout=core.Duration.hours(5)) ################################################################################ # Set up job submission Lambda lambda_role = iam.Role( self, 'UmccriseLambdaRole', assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AWSLambdaBasicExecutionRole'), iam.ManagedPolicy.from_aws_managed_policy_name( 'AWSBatchFullAccess') # TODO: restrict! ]) for bucket in ro_buckets: bucket.grant_read(lambda_role) for bucket in rw_buckets: bucket.grant_read(lambda_role) # TODO: support dev/prod split, i.e. image being configurable on dev, but fixed on prod # may need a default JobDefinition to be set up lmbda.Function(self, 'UmccriseLambda', function_name='umccrise_batch_lambda', handler='umccrise.lambda_handler', runtime=lmbda.Runtime.PYTHON_3_7, code=lmbda.Code.from_asset('lambdas/umccrise'), environment={ 'JOBNAME_PREFIX': "UMCCRISE_", 'JOBQUEUE': job_queue.job_queue_name, 'REFDATA_BUCKET': props['refdata_bucket'], 'DATA_BUCKET': props['data_bucket'], 'UMCCRISE_MEM': '50000', 'UMCCRISE_VCPUS': '16' }, role=lambda_role)
def __init__(self, scope: core.Construct, **kwargs) -> None: self.deploy_env = active_environment super().__init__(scope, id=f"{self.deploy_env.value}-databricks-stack", **kwargs) cross_account_role = iam.Role( self, id=f"iam-{self.deploy_env.value}-databricks-cross-account-role", assumed_by=iam.AccountPrincipal(account_id="874361926784433"), description=f"Allows databricks access to account", ) cross_account_policy = iam.Policy( self, id=f"iam-{self.deploy_env.value}-databricks-cross-account-policy", policy_name= f"iam-{self.deploy_env.value}-databricks-cross-account-policy", statements=[ iam.PolicyStatement( actions=[ "ec2:AssociateDhcpOptions", "ec2:AssociateIamInstanceProfile", "ec2:AssociateRouteTable", "ec2:AttachInternetGateway", "ec2:AttachVolume", "ec2:AuthorizeSecurityGroupEgress", "ec2:AuthorizeSecurityGroupIngress", "ec2:CancelSpotInstanceRequests", "ec2:CreateDhcpOptions", "ec2:CreateInternetGateway", "ec2:CreateKeyPair", "ec2:CreatePlacementGroup", "ec2:CreateRoute", "ec2:CreateSecurityGroup", "ec2:CreateSubnet", "ec2:CreateTags", "ec2:CreateVolume", "ec2:CreateVpc", "ec2:CreateVpcPeeringConnection", "ec2:DeleteInternetGateway", "ec2:DeleteKeyPair", "ec2:DeletePlacementGroup", "ec2:DeleteRoute", "ec2:DeleteRouteTable", "ec2:DeleteSecurityGroup", "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DeleteVolume", "ec2:DeleteVpc", "ec2:DescribeAvailabilityZones", "ec2:DescribeIamInstanceProfileAssociations", "ec2:DescribeInstanceStatus", "ec2:DescribeInstances", "ec2:DescribePlacementGroups", "ec2:DescribePrefixLists", "ec2:DescribeReservedInstancesOfferings", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSpotInstanceRequests", "ec2:DescribeSpotPriceHistory", "ec2:DescribeSubnets", "ec2:DescribeVolumes", "ec2:DescribeVpcs", "ec2:DetachInternetGateway", "ec2:DisassociateIamInstanceProfile", "ec2:ModifyVpcAttribute", "ec2:ReplaceIamInstanceProfileAssociation", "ec2:RequestSpotInstances", "ec2:RevokeSecurityGroupEgress", "ec2:RevokeSecurityGroupIngress", "ec2:RunInstances", "ec2:TerminateInstances", ], resources=["*"], ), iam.PolicyStatement( actions=[ "iam:CreateServiceLinkedRole", "iam:PutRolePolicy" ], resources=[ "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot" ], conditions={ "StringEquals": { "iam:AWSServiceName": "spot.amazonaws.com" } }, ), iam.PolicyStatement( actions=[ "s3:GetBucketNotification", "s3:PutBucketNotification", "sns:ListSubscriptionsByTopic", "sns:GetTopicAttributes", "sns:SetTopicAttributes", "sns:CreateTopic", "sns:TagResource", "sns:Publish", "sns:Subscribe", "sqs:CreateQueue", "sqs:DeleteMessage", "sqs:DeleteMessageBatch", "sqs:ReceiveMessage", "sqs:SendMessage", "sqs:GetQueueUrl", "sqs:GetQueueAttributes", "sqs:SetQueueAttributes", "sqs:TagQueue", "sqs:ChangeMessageVisibility", "sqs:ChangeMessageVisibilityBatch", ], resources=[ f"arn:aws:s3:::s3-juan-armond-{self.deploy_env.value}-data-lake-*", f"arn:aws:sqs:{self.region}:{self.account}:databricks-auto-ingest-*", f"arn:aws:sns:{self.region}:{self.account}:databricks-auto-ingest-*", ], ), iam.PolicyStatement( actions=[ "sqs:ListQueues", "sqs:ListQueueTags", "sns:ListTopics" ], resources=["*"], ), iam.PolicyStatement( actions=[ "sns:Unsubscribe", "sns:DeleteTopic", "sqs:DeleteQueue" ], resources=[ f"arn:aws:sqs:{self.region}:{self.account}:databricks-auto-ingest-*", f"arn:aws:sns:{self.region}:{self.account}:databricks-auto-ingest-*", ], ), ], ) cross_account_role.attach_inline_policy(cross_account_policy) bucket = s3.Bucket( self, id=f"s3-{self.deploy_env.value}-juan-armond-databricks-bucket", bucket_name= f"s3-{self.deploy_env.value}-juan-armond-databricks-bucket", ) bucket.add_to_resource_policy( iam.PolicyStatement( principals=[iam.AccountPrincipal(account_id="414351767826")], effect=iam.Effect.ALLOW, actions=[ "s3:GetObject", "s3:GetObjectVersion", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket", "s3:GetBucketLocation", ], resources=[bucket.bucket_arn, bucket.bucket_arn + "/*"], )) access_role = iam.Role( self, id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-role", assumed_by=iam.ServicePrincipal("ec2"), description=f"Allows databricks access to data lake", ) access_policy = iam.Policy( self, id= f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy", policy_name= f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy", statements=[ iam.PolicyStatement( actions=[ "s3:ListBucket", "s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:PutObjectAcl", ], resources=[ f"arn:aws:s3:::s3-juan-armond-{self.deploy_env.value}-data-lake-*", f"arn:aws:s3:::s3-juan-armond-{self.deploy_env.value}-data-lake-*/*", ], ), iam.PolicyStatement( actions=[ "iam:CreateServiceLinkedRole", "iam:PutRolePolicy" ], resources=[ "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot" ], conditions={ "StringEquals": { "iam:AWSServiceName": "spot.amazonaws.com" } }, ), iam.PolicyStatement( actions=[ "glue:BatchCreatePartition", "glue:BatchDeletePartition", "glue:BatchGetPartition", "glue:CreateDatabase", "glue:CreateTable", "glue:CreateUserDefinedFunction", "glue:DeleteDatabase", "glue:DeletePartition", "glue:DeleteTable", "glue:DeleteUserDefinedFunction", "glue:GetDatabase", "glue:GetDatabases", "glue:GetPartition", "glue:GetPartitions", "glue:GetTable", "glue:GetTables", "glue:GetUserDefinedFunction", "glue:GetUserDefinedFunctions", "glue:UpdateDatabase", "glue:UpdatePartition", "glue:UpdateTable", "glue:UpdateUserDefinedFunction", ], resources=["*"], ), ], ) access_role.attach_inline_policy(access_policy) cross_account_policy_data_access = iam.Policy( self, id= f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access", policy_name= f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access", statements=[ iam.PolicyStatement( actions=[ "iam:PassRole", ], resources=[access_role.role_arn], ) ], ) iam.CfnInstanceProfile( self, id= f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile", instance_profile_name= f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile", roles=[access_role.role_name], ) cross_account_role.attach_inline_policy( cross_account_policy_data_access)
def __init__(self, scope: core.Construct, **kwargs) -> None: self.deploy_env = active_environment super().__init__(scope, id=f"{self.deploy_env.value}-databricks-stack", **kwargs) cross_account_role = iam.Role( self, id=f"iam-{self.deploy_env.value}-databricks-cross-account-role", assumed_by=iam.AccountPrincipal(account_id="414351767826"), external_ids=["88086fcd-a382-4d73-9ddf-7e12326a3584"], description=f"Allows databricks access to account", ) cross_account_policy = iam.Policy( self, id=f"iam-{self.deploy_env.value}-databricks-cross-account-policy", policy_name=f"iam-{self.deploy_env.value}-databricks-cross-account-policy", statements=[ iam.PolicyStatement( actions=[ "ec2:AllocateAddress", "ec2:AssociateDhcpOptions", "ec2:AssociateIamInstanceProfile", "ec2:AssociateRouteTable", "ec2:AttachInternetGateway", "ec2:AttachVolume", "ec2:AuthorizeSecurityGroupEgress", "ec2:AuthorizeSecurityGroupIngress", "ec2:CancelSpotInstanceRequests", "ec2:CreateDhcpOptions", "ec2:CreateInternetGateway", "ec2:CreateKeyPair", "ec2:CreateNatGateway", "ec2:CreatePlacementGroup", "ec2:CreateRoute", "ec2:CreateRouteTable", "ec2:CreateSecurityGroup", "ec2:CreateSubnet", "ec2:CreateTags", "ec2:CreateVolume", "ec2:CreateVpc", "ec2:CreateVpcEndpoint", "ec2:DeleteDhcpOptions", "ec2:DeleteInternetGateway", "ec2:DeleteKeyPair", "ec2:DeleteNatGateway", "ec2:DeletePlacementGroup", "ec2:DeleteRoute", "ec2:DeleteRouteTable", "ec2:DeleteSecurityGroup", "ec2:DeleteSubnet", "ec2:DeleteTags", "ec2:DeleteVolume", "ec2:DeleteVpc", "ec2:DeleteVpcEndpoints", "ec2:DescribeAvailabilityZones", "ec2:DescribeIamInstanceProfileAssociations", "ec2:DescribeInstanceStatus", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", "ec2:DescribeNatGateways", "ec2:DescribePlacementGroups", "ec2:DescribePrefixLists", "ec2:DescribeReservedInstancesOfferings", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSpotInstanceRequests", "ec2:DescribeSpotPriceHistory", "ec2:DescribeSubnets", "ec2:DescribeVolumes", "ec2:DescribeVpcs", "ec2:DetachInternetGateway", "ec2:DisassociateIamInstanceProfile", "ec2:DisassociateRouteTable", "ec2:ModifyVpcAttribute", "ec2:ReleaseAddress", "ec2:ReplaceIamInstanceProfileAssociation", "ec2:RequestSpotInstances", "ec2:RevokeSecurityGroupEgress", "ec2:RevokeSecurityGroupIngress", "ec2:RunInstances", "ec2:TerminateInstances", ], resources=["*"], ), iam.PolicyStatement( actions=["iam:CreateServiceLinkedRole", "iam:PutRolePolicy"], resources=[ "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot" ], conditions={ "StringLike": {"iam:AWSServiceName": "spot.amazonaws.com"} }, ), ], ) cross_account_role.attach_inline_policy(cross_account_policy) bucket = s3.Bucket( self, id=f"s3-{self.deploy_env.value}-belisquito-databricks-bucket-turma-5", bucket_name=f"s3-{self.deploy_env.value}-belisquito-databricks-bucket-turma-5", ) bucket.add_to_resource_policy( iam.PolicyStatement( principals=[iam.AccountPrincipal(account_id="414351767826")], effect=iam.Effect.ALLOW, actions=[ "s3:GetObject", "s3:GetObjectVersion", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket", "s3:GetBucketLocation", ], resources=[bucket.bucket_arn, bucket.bucket_arn + "/*"], ) ) access_role = iam.Role( self, id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-role", assumed_by=iam.ServicePrincipal("ec2"), description=f"Allows databricks access to data lake", ) access_policy = iam.Policy( self, id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy", policy_name=f"iam-{self.deploy_env.value}-databricks-data-lake-access-policy", statements=[ iam.PolicyStatement( actions=[ "s3:ListBucket", "s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:PutObjectAcl", ], resources=[ f"arn:aws:s3:::s3-belisquito-turma-5-{self.deploy_env.value}-data-lake-*", f"arn:aws:s3:::s3-belisquito-turma-5-{self.deploy_env.value}-data-lake-*/*", ], ), iam.PolicyStatement( actions=["iam:CreateServiceLinkedRole", "iam:PutRolePolicy"], resources=[ "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot" ], conditions={ "StringEquals": {"iam:AWSServiceName": "spot.amazonaws.com"} }, ), iam.PolicyStatement( actions=[ "glue:BatchCreatePartition", "glue:BatchDeletePartition", "glue:BatchGetPartition", "glue:CreateDatabase", "glue:CreateTable", "glue:CreateUserDefinedFunction", "glue:DeleteDatabase", "glue:DeletePartition", "glue:DeleteTable", "glue:DeleteUserDefinedFunction", "glue:GetDatabase", "glue:GetDatabases", "glue:GetPartition", "glue:GetPartitions", "glue:GetTable", "glue:GetTables", "glue:GetUserDefinedFunction", "glue:GetUserDefinedFunctions", "glue:UpdateDatabase", "glue:UpdatePartition", "glue:UpdateTable", "glue:UpdateUserDefinedFunction", ], resources=["*"], ), ], ) access_role.attach_inline_policy(access_policy) cross_account_policy_data_access = iam.Policy( self, id=f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access", policy_name=f"iam-{self.deploy_env.value}-databricks-cross-account-policy-data-access", statements=[ iam.PolicyStatement( actions=[ "iam:PassRole", ], resources=[access_role.role_arn], ) ], ) iam.CfnInstanceProfile( self, id=f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile", instance_profile_name=f"iam-{self.deploy_env.value}-databricks-data-lake-access-instance-profile", roles=[access_role.role_name], ) cross_account_role.attach_inline_policy(cross_account_policy_data_access)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) eks_vpc = ec2.Vpc( self, "VPC", cidr="10.0.0.0/16" ) # Create IAM Role For EC2 bastion instance to be able to manage the cluster bastion_role = iam.Role( self, "BastionRole", assumed_by=iam.CompositePrincipal( iam.ServicePrincipal("ec2.amazonaws.com"), iam.AccountRootPrincipal() ) ) self.bastion_role = bastion_role # Create EC2 Instance Profile for that Role instance_profile = iam.CfnInstanceProfile( self, "InstanceProfile", roles=[bastion_role.role_name] ) # Create SecurityGroup for the Control Plane ENIs eks_security_group = ec2.SecurityGroup( self, "EKSSecurityGroup", vpc=eks_vpc, allow_all_outbound=True ) eks_security_group.add_ingress_rule( ec2.Peer.ipv4('10.0.0.0/16'), ec2.Port.all_traffic() ) # Create an EKS Cluster eks_cluster = eks.Cluster( self, "cluster", vpc=eks_vpc, masters_role=bastion_role, default_capacity_type=eks.DefaultCapacityType.NODEGROUP, default_capacity_instance=ec2.InstanceType("m5.large"), default_capacity=2, security_group=eks_security_group, endpoint_access=eks.EndpointAccess.PUBLIC_AND_PRIVATE, version=eks.KubernetesVersion.V1_18 ) # Deploy ALB Ingress Controller # Create the k8s Service account and corresponding IAM Role mapped via IRSA alb_service_account = eks_cluster.add_service_account( "aws-load-balancer-controller", name="aws-load-balancer-controller", namespace="kube-system" ) # Create the PolicyStatements to attach to the role # I couldn't find a way to get this to work with a PolicyDocument and there are 10 of these alb_policy_statement_json_1 = { "Effect": "Allow", "Action": [ "acm:DescribeCertificate", "acm:ListCertificates", "acm:GetCertificate" ], "Resource": "*" } alb_policy_statement_json_2 = { "Effect": "Allow", "Action": [ "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateSecurityGroup", "ec2:CreateTags", "ec2:DeleteTags", "ec2:DeleteSecurityGroup", "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeInstances", "ec2:DescribeInstanceStatus", "ec2:DescribeInternetGateways", "ec2:DescribeNetworkInterfaces", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeTags", "ec2:DescribeVpcs", "ec2:ModifyInstanceAttribute", "ec2:ModifyNetworkInterfaceAttribute", "ec2:RevokeSecurityGroupIngress" ], "Resource": "*" } alb_policy_statement_json_3 = { "Effect": "Allow", "Action": [ "elasticloadbalancing:AddListenerCertificates", "elasticloadbalancing:AddTags", "elasticloadbalancing:CreateListener", "elasticloadbalancing:CreateLoadBalancer", "elasticloadbalancing:CreateRule", "elasticloadbalancing:CreateTargetGroup", "elasticloadbalancing:DeleteListener", "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:DeleteRule", "elasticloadbalancing:DeleteTargetGroup", "elasticloadbalancing:DeregisterTargets", "elasticloadbalancing:DescribeListenerCertificates", "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeRules", "elasticloadbalancing:DescribeSSLPolicies", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:ModifyListener", "elasticloadbalancing:ModifyLoadBalancerAttributes", "elasticloadbalancing:ModifyRule", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:RegisterTargets", "elasticloadbalancing:RemoveListenerCertificates", "elasticloadbalancing:RemoveTags", "elasticloadbalancing:SetIpAddressType", "elasticloadbalancing:SetSecurityGroups", "elasticloadbalancing:SetSubnets", "elasticloadbalancing:SetWebAcl" ], "Resource": "*" } alb_policy_statement_json_4 = { "Effect": "Allow", "Action": [ "iam:CreateServiceLinkedRole", "iam:GetServerCertificate", "iam:ListServerCertificates" ], "Resource": "*" } alb_policy_statement_json_5 = { "Effect": "Allow", "Action": [ "cognito-idp:DescribeUserPoolClient" ], "Resource": "*" } alb_policy_statement_json_6 = { "Effect": "Allow", "Action": [ "waf-regional:GetWebACLForResource", "waf-regional:GetWebACL", "waf-regional:AssociateWebACL", "waf-regional:DisassociateWebACL" ], "Resource": "*" } alb_policy_statement_json_7 = { "Effect": "Allow", "Action": [ "tag:GetResources", "tag:TagResources" ], "Resource": "*" } alb_policy_statement_json_8 = { "Effect": "Allow", "Action": [ "waf:GetWebACL" ], "Resource": "*" } alb_policy_statement_json_9 = { "Effect": "Allow", "Action": [ "wafv2:GetWebACL", "wafv2:GetWebACLForResource", "wafv2:AssociateWebACL", "wafv2:DisassociateWebACL" ], "Resource": "*" } alb_policy_statement_json_10 = { "Effect": "Allow", "Action": [ "shield:DescribeProtection", "shield:GetSubscriptionState", "shield:DeleteProtection", "shield:CreateProtection", "shield:DescribeSubscription", "shield:ListProtections" ], "Resource": "*" } # Attach the necessary permissions alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_1)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_2)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_3)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_4)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_5)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_6)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_7)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_8)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_9)) alb_service_account.add_to_policy(iam.PolicyStatement.from_json(alb_policy_statement_json_10)) # Deploy the ALB Ingress Controller from the Helm chart eks_cluster.add_helm_chart( "aws-load-balancer-controller", chart="aws-load-balancer-controller", repository="https://aws.github.io/eks-charts", namespace="kube-system", values={ "clusterName": eks_cluster.cluster_name, "region": self.region, "vpcId": eks_vpc.vpc_id, "serviceAccount": { "create": False, "name": "aws-load-balancer-controller" } } ) # Deploy External DNS Controller # Create the k8s Service account and corresponding IAM Role mapped via IRSA externaldns_service_account = eks_cluster.add_service_account( "external-dns", name="external-dns", namespace="kube-system" ) # Create the PolicyStatements to attach to the role externaldns_policy_statement_json_1 = { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets" ], "Resource": [ "arn:aws:route53:::hostedzone/*" ] } externaldns_policy_statement_json_2 = { "Effect": "Allow", "Action": [ "route53:ListHostedZones", "route53:ListResourceRecordSets" ], "Resource": [ "*" ] } # Add the policies to the service account externaldns_service_account.add_to_policy(iam.PolicyStatement.from_json(externaldns_policy_statement_json_1)) externaldns_service_account.add_to_policy(iam.PolicyStatement.from_json(externaldns_policy_statement_json_2)) # Deploy the Helm Chart eks_cluster.add_helm_chart( "external-dns", chart="external-dns", repository="https://charts.bitnami.com/bitnami", namespace="kube-system", values={ "provider": "aws", "aws": { "region": self.region }, "serviceAccount": { "create": False, "name": "external-dns" }, "podSecurityContext": { "fsGroup": 65534 } } ) # Install external secrets controller # Create the Service Account externalsecrets_service_account = eks_cluster.add_service_account( "kubernetes-external-secrets", name="kubernetes-external-secrets", namespace="kube-system" ) # Define the policy in JSON externalsecrets_policy_statement_json_1 = { "Effect": "Allow", "Action": [ "secretsmanager:GetResourcePolicy", "secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret", "secretsmanager:ListSecretVersionIds" ], "Resource": [ "*" ] } # Add the policies to the service account externalsecrets_service_account.add_to_policy(iam.PolicyStatement.from_json(externalsecrets_policy_statement_json_1)) # Deploy the Helm Chart eks_cluster.add_helm_chart( "external-secrets", chart="kubernetes-external-secrets", repository="https://external-secrets.github.io/kubernetes-external-secrets/", namespace="kube-system", values={ "env": { "AWS_REGION": self.region }, "serviceAccount": { "name": "kubernetes-external-secrets", "create": False }, "securityContext": { "fsGroup": 65534 } } ) # Deploy Flux # Deploy the Helm Chart eks_cluster.add_helm_chart( "flux", chart="flux", repository="https://charts.fluxcd.io", namespace="kube-system", values={ "git": { "url": "[email protected]:jasonumiker/k8s-plus-aws-gitops", "path": "k8s-app-resources", "branch": "master" } } ) # Deploy Prometheus and Grafana # TODO Replace this with the new AWS Managed Prometheus and Grafana when available eks_cluster.add_helm_chart( "metrics", chart="kube-prometheus-stack", repository="https://prometheus-community.github.io/helm-charts", namespace="monitoring", values={ "prometheus": { "prometheusSpec": { "storageSpec": { "volumeClaimTemplate": { "spec": { "accessModes": [ "ReadWriteOnce" ], "resources": { "requests": { "storage": "8Gi" } }, "storageClassName": "gp2" } } } } }, "alertmanager": { "alertmanagerSpec": { "storage": { "volumeClaimTemplate": { "spec": { "accessModes": [ "ReadWriteOnce" ], "resources": { "requests": { "storage": "2Gi" } }, "storageClassName": "gp2" } } } } }, "grafana": { "persistence": { "enabled": "true", "storageClassName": "gp2" } } } ) # Deploy Fluentbit and Elasticsearch # Deploy an ElasticSearch Domain es_domain = es.Domain( self, "ESDomain", version=es.ElasticsearchVersion.V7_9 ) # Create the Service Account fluentbit_service_account = eks_cluster.add_service_account( "fluentbit", name="fluentbit", namespace="monitoring" ) # Define the policy in JSON fluentbit_policy_statement_json_1 = { "Effect": "Allow", "Action": [ "es:ESHttp*" ], "Resource": [ es_domain.domain_arn ] } # Add the policies to the service account fluentbit_service_account.add_to_policy(iam.PolicyStatement.from_json(externalsecrets_policy_statement_json_1)) # Grant fluentbit access to our ES Domain es_domain.grant_write(fluentbit_service_account) eks_cluster.add_helm_chart( "fluent-bit", chart="fluent-bit", repository="https://fluent.github.io/helm-charts", namespace="monitoring", values={ "serviceAccount": { "create": False, "name": "fluentbit" }, "config": { "outputs": "[OUTPUT]\n Name es\n Match *\n Host "+es_domain.domain_endpoint+"\n Port 443\n TLS On\n AWS_Auth On\n AWS_Region "+self.region+"\n Retry_Limit 6\n", } } )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # ======================== # VPC # ======================== # VPC vpc = ec2.Vpc( self, 'fetch-and-run-vpc', max_azs=2, subnet_configuration=[ ec2.SubnetConfiguration( name='public-subnet', subnet_type=ec2.SubnetType.PUBLIC ) ], nat_gateways=0 ) # Security Group sg = ec2.SecurityGroup( self, 'fetch-and-run-sg', vpc=vpc, description='SG for fetch and run', security_group_name='fetch-and-run-sg' ) # Ingress from IP address via HTTP, SSH for port in PORTS: sg.add_ingress_rule( peer=ec2.Peer.ipv4(IP_ADDRESS), connection=ec2.Port.tcp(port) ) # ======================== # IAM # ======================== ''' I. Batch Instance Role - Makes calls to other AWS services on your behalf to manage the resources that you use with the service ''' batch_service_role = iam.Role.from_role_arn( self, 'batch-service-role', role_arn=BATCH_SERVICE_ROLE_ARN ) ''' II. ECS Instance Role - Batch compute environmens are populated with ECS container instances, which run the ECS container agent locally - ECS container agent makes calls to AWS APIs on your behalf - Container instances that run the agent require a policy and role for these services to know that the agent belongs to you - Instance Profile uses the batch instance role name - This is fed into the compute environment ''' batch_instance_role = iam.Role.from_role_arn( self, 'batch-instance-role', role_arn=ECS_INSTANCE_ROLE_ARN ) instance_profile = iam.CfnInstanceProfile( self, 'instance-profile', roles=[batch_instance_role.role_name] ) ''' Job Role - Used in the job definition - IAM role that the container can assume for AWS permissions When the fetch_and_run image runs as an AWS Batch job, it fetches the job script from Amazon S3. You need an IAM role that the AWS Batch job can use to access S3 Trusted Entity --> AWS service --> Elastic Container Service --> Elastic Container Service Task - In the Role's trust relationship, this will be displayed as follows: { "Version": "2012-10-17", "Statement": [ { "Sid": "", "Effect": "Allow", "Principal": { "Service": "ecs-tasks.amazonaws.com" }, "Action": "sts:AssumeRole" } ] } Default is for a role to be created ''' batch_job_role = iam.Role.from_role_arn( self, 'batch-job-role', role_arn=BATCH_JOB_ROLE_ARN ) # ======================== # ECR # ======================== ''' Repository TODO: Evaluate integrating repository into CDK (in this stack or another) ''' ecr_repository = ecr.Repository.from_repository_name( self, 'ecr-repository', repository_name=ECR_REPOSITORY_NAME ) ''' Container Image NOTE: We are pulling the image directly from ECR. Pushed before stack is created. - Can alternatively create the image from files in the stack (commented out) TODO: Evaluate ability to programatically update the tag. - Manually updating the tag follows approach of pushing image before stack creation/updates - Review adding alphanumeric tag as opposed to simply 'latest' --> more detail for auditing ''' # image_asset = ecr_assets.DockerImageAsset( # self, 'docker-image', # directory='./fetch-and-run', # file='./Dockerfile' # ) # image = ecs.ContainerImage.from_docker_image_asset(image_asset) image = ecs.ContainerImage.from_ecr_repository( repository=ecr_repository, tag='latest' ) # ======================== # BATCH # ======================== ''' I. Compute Environment - Execution runtime of submitted batch jobs ''' compute_environment = batch.ComputeEnvironment( self, 'batch-compute-environment', compute_environment_name='batch-compute-environment', compute_resources=batch.ComputeResources( vpc=vpc, # BEST_FIT_PROGRESSIVE will select an additional instance type that is large enough to meet the requirements of the jobs in the queue, with a preference for an instance type with a lower cost. allocation_strategy=batch.AllocationStrategy.BEST_FIT_PROGRESSIVE, compute_resources_tags={ "name": "fetch-and-run" }, ec2_key_pair=KEY_PAIR, instance_role=instance_profile.attr_arn, security_groups=[sg], type=batch.ComputeResourceType.ON_DEMAND, vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PUBLIC) ), service_role=batch_service_role, ) ''' II. Job Queue - Queue where batch jobs can be submitted ''' job_queue = batch.JobQueue( self, 'fetch-and-run-queue', compute_environments=[ batch.JobQueueComputeEnvironment( compute_environment=compute_environment, order=1 )], job_queue_name='fetch-and-run-queue' ) ''' III. Job Definition - Group various job properties (image, resource requirements, env variables) into a single definition. Definitionns are used to job submission time TODO: Build out functionality for the following: - `command` => The command that is passed to the container. If you provide a shell command as a single string, you have to quote command-line arguments - `environment` => The environment variables to pass to the container - `mount_points` => The mount points for data volumes in your container - `volumes` => A list of data volumes used in a job. NOTE: Can optionally add command, environment variables directly in code - Alternatively can reference them in `fetch_and_run.sh` ''' job_definition = batch.JobDefinition( self, 'fetch-and-run-job-definition', container=batch.JobDefinitionContainer( image=image, job_role=batch_job_role, # The hard limit (in MiB) of memory to present to the container memory_limit_mib=500, # The number of vCPUs reserved for the container. Each vCPU is equivalent to 1,024 CPU vcpus=1, user="******" ) )
def __init__(self, scope: core.Construct, data_lake: DataLake, common: Common, **kwargs) -> None: self.env = data_lake.env.value super().__init__(scope, id=f'{self.env}-emr-transform', **kwargs) self.logs_bucket = s3.Bucket( self, f'{self.env}-emr-logs-bucket', bucket_name=f's3-belisco-{self.env}-emr-logs-bucket', removal_policy=core.RemovalPolicy.DESTROY) buckets_arns = [ data_lake.data_lake_raw_bucket.bucket_arn, data_lake.data_lake_processed_bucket.bucket_arn, data_lake.data_lake_curated_bucket.bucket_arn ] self.datalake_emr_policy = iam.Policy( self, id=f'iam-{self.env}-emr-data-lake', policy_name=f'iam-{self.env}-emr-data-lake', statements=[ iam.PolicyStatement(actions=[ 's3:*', ], resources=buckets_arns + [f'{arn}/*' for arn in buckets_arns]) ]) self.emr_role = iam.Role( self, f'{self.env}-emr-cluster-role', assumed_by=iam.ServicePrincipal('elasticmapreduce.amazonaws.com'), description='Role to allow EMR to process data', managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonElasticMapReduceRole') ]) self.emr_role.attach_inline_policy(self.datalake_emr_policy) self.emr_ec2_role = iam.Role( self, f'{self.env}-emr-ec2-role', assumed_by=iam.ServicePrincipal('ec2.amazonaws.com'), description='Role to allow EMR to process data', managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonElasticMapReduceforEC2Role') ]) self.emr_ec2_role.attach_inline_policy(self.datalake_emr_policy) self.emr_ec2_instance_profile = iam.CfnInstanceProfile( self, f'{self.env}-emr-instance_profile', instance_profile_name=f'{self.env}-emr-instance_profile', roles=[self.emr_ec2_role.role_name]) self.cluster = emr.CfnCluster( self, f'{self.env}-emr-cluster', name=f'{self.env}-emr-cluster', instances=emr.CfnCluster.JobFlowInstancesConfigProperty( master_instance_group=emr.CfnCluster. InstanceGroupConfigProperty(instance_count=1, instance_type='m4.large', market='ON_DEMAND', name='Master'), core_instance_group=emr.CfnCluster.InstanceGroupConfigProperty( instance_count=2, instance_type='m4.large', market='ON_DEMAND', name='Core'), termination_protected=False, ec2_subnet_id=common.custom_vpc.private_subnets[0].subnet_id), applications=[emr.CfnCluster.ApplicationProperty(name='Spark')], log_uri=f's3://{self.logs_bucket.bucket_name}/logs', job_flow_role=self.emr_ec2_instance_profile.get_att( 'Arn').to_string(), service_role=self.emr_role.role_arn, release_label='emr-5.30.1', visible_to_all_users=True, configurations=[ emr.CfnCluster.ConfigurationProperty( classification='spark-hive-site', configuration_properties={ "hive.metastore.client.factory.class": "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory" }) ])