def __create_pull_service(self, service_name: str, ctx: object): ctx_srv = getattr(ctx.inbound.services.pull, service_name) ecs_task_role = self.__create_default_task_role(service_name) log_driver = ecs.LogDriver.aws_logs(log_group=self.log_group, stream_prefix=service_name) # create a Fargate task definition task_definition = ecs.FargateTaskDefinition( scope=self, id=f"{service_name}_task_definition", cpu=ctx_srv.size.cpu, memory_limit_mib=ctx_srv.size.ram, execution_role=self.ecs_exec_role, task_role=ecs_task_role, ) # create a container definition and associate with the Fargate task container_vars = self.__get_container_vars(service_name, ctx, ctx_srv) container = ecs.ContainerDefinition( scope=self, id=f"{service_name}_container_definition", task_definition=task_definition, image=ecs.ContainerImage.from_ecr_repository( self.ecr_repository, "latest"), logging=log_driver, **container_vars) security_group = ec2.SecurityGroup(scope=self, id=f"{service_name}_sg", vpc=self.vpc) service = ecs.FargateService(scope=self, id=f"{service_name}_service", task_definition=task_definition, cluster=self.cluster, desired_count=getattr( ctx_srv, "desired_count", ctx.default_desired_count), service_name=service_name, security_group=security_group) scaling = service.auto_scale_task_count( max_capacity=ctx_srv.scaling.max_capacity, min_capacity=ctx_srv.scaling.min_capacity) scaling.scale_on_cpu_utilization( id="cpu_scaling", target_utilization_percent=ctx_srv.scaling. target_utilization_percent, scale_in_cooldown=core.Duration.seconds( ctx_srv.scaling.scale_in_cooldown_seconds), scale_out_cooldown=core.Duration.seconds( ctx_srv.scaling.scale_out_cooldown_seconds))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create the VPC for the honeypot(s), default is all AZs in region vpc = ec2.Vpc(self, "HoneypotVpc", max_azs=3) # Create the ECS cluster where fargate can deploy the Docker containers cluster = ecs.Cluster(self, "HoneypotCluster", vpc=vpc) # Define task definition for Fargate Service task_definition = ecs.FargateTaskDefinition(self, "HoneypotTasks", cpu=256, memory_limit_mib=512) # Container definition container_definition = ecs.ContainerDefinition( self, "HoneypotContainerDefinition", image=ecs.ContainerImage.from_registry("statixs/cowrie"), #image=ecs.ContainerImage.from_asset(directory = "docker"), task_definition=task_definition, stop_timeout=core.Duration.seconds(2), logging=ecs.AwsLogDriver( stream_prefix="cowrie", log_retention=logs.RetentionDays.ONE_WEEK, ), ) # ECS Security Group definition sg_ssh = ec2.SecurityGroup(self, "honeypot-sg-ssh", vpc=vpc, description="Allow SSH to the honeypot") sg_ssh.add_ingress_rule(ec2.Peer.ipv4("0.0.0.0/0"), ec2.Port.tcp(22)) # Fargate service definition fargate_service = ecs.FargateService( self, "HoneypotFargate", cluster=cluster, assign_public_ip=True, desired_count=1, security_group=sg_ssh, task_definition=task_definition, platform_version=ecs.FargatePlatformVersion.VERSION1_4)
def __init__(self, scope: core.Construct, id: str, ctx: object, ecr_repository: ecr.Repository, kinesis_stream: ks.Stream, state_table: ddb.Table, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.ecr_repository = ecr_repository self.kinesis_stream = kinesis_stream self.state_table = state_table service_name = "processor" ctx_srv = getattr(ctx.outbound.services.pull, service_name) self.vpc = ec2.Vpc.from_vpc_attributes( self, "VPC", **ctx.vpc_props.dict() ) # CloudWatch Logs Group self.log_group = cwl.LogGroup( scope = self, id = "logs" ) # Create a new ECS cluster for our services self.cluster = ecs.Cluster( self, vpc = self.vpc, id = f"{id}_cluster" ) cluster_name_output = core.CfnOutput( scope=self, id="cluster-name-out", value=self.cluster.cluster_name, export_name=f"{id}-cluster-name" ) service_names_output = core.CfnOutput( scope=self, id="service-names-out", value=service_name, export_name=f"{id}-service-names" ) # Create a role for ECS to interact with AWS APIs with standard permissions self.ecs_exec_role = iam.Role( scope = self, id = "ecs_logstash-exec_role", assumed_by = iam.ServicePrincipal("ecs-tasks.amazonaws.com"), managed_policies = ([ iam.ManagedPolicy.from_aws_managed_policy_name( "service-role/AmazonECSTaskExecutionRolePolicy") ]) ) # Grant ECS additional permissions to decrypt secrets from Secrets Manager that have been encrypted with our custom key if getattr(ctx, "secrets_key_arn", None) is not None: self.ecs_exec_role.add_to_policy( iam.PolicyStatement( actions = ["kms:Decrypt"], effect = iam.Effect.ALLOW, resources = [ctx.secrets_key_arn] )) # Grant ECS permissions to log to our log group self.log_group.grant_write(self.ecs_exec_role) # Create a task role to grant permissions for Logstash to interact with AWS APIs ecs_task_role = iam.Role( scope = self, id = f"{service_name}_task_role", assumed_by = iam.ServicePrincipal("ecs-tasks.amazonaws.com") ) # Add permissions for Logstash to send metrics to CloudWatch ecs_task_role.add_to_policy( iam.PolicyStatement( actions = ["cloudwatch:PutMetricData"], effect = iam.Effect.ALLOW, resources = ["*"] )) # Add permissions for Logstash to interact with our Kinesis queue self.kinesis_stream.grant_read(ecs_task_role) # Remove this when next version of kinesis module is released # https://github.com/aws/aws-cdk/pull/6141 ecs_task_role.add_to_policy( iam.PolicyStatement( actions = ["kinesis:ListShards"], effect = iam.Effect.ALLOW, resources = [self.kinesis_stream.stream_arn] )) # Add permissions for Logstash to store Kinesis Consumer Library (KCL) state tracking in DynamoDB state_table.grant_full_access(ecs_task_role) # Add permissions for Logstash to upload logs to S3 for archive bucket_resources = [] for k, v in ctx_srv.variables.items(): if k.endswith("_log_bucket"): bucket_resources.append('arn:aws:s3:::{0}'.format(v)) bucket_resources.append('arn:aws:s3:::{0}/*'.format(v)) ecs_task_role.add_to_policy( iam.PolicyStatement( actions=[ "s3:PutObject", "s3:ListMultipartUploadParts", "s3:ListBucket", "s3:AbortMultipartUpload" ], effect=iam.Effect.ALLOW, resources=bucket_resources )) # Task Definition task_definition = ecs.FargateTaskDefinition( scope = self, id = f"{service_name}_task_definition", cpu = ctx_srv.size.cpu, memory_limit_mib = ctx_srv.size.ram, execution_role = self.ecs_exec_role, task_role = ecs_task_role, ) log_driver = ecs.LogDriver.aws_logs( log_group = self.log_group, stream_prefix = service_name) # Container Definition container_vars = self.__get_container_vars(service_name, ctx, ctx_srv) container = ecs.ContainerDefinition( scope = self, id = f"{service_name}_container_definition", task_definition = task_definition, image = ecs.ContainerImage.from_ecr_repository(self.ecr_repository, "latest"), logging = log_driver, **container_vars ) # Service Definition security_group = ec2.SecurityGroup( scope = self, id = f"{service_name}_sg", vpc = self.vpc ) service = ecs.FargateService( scope = self, id = f"{service_name}_fargate_service", task_definition = task_definition, cluster = self.cluster, desired_count = getattr(ctx_srv, "desired_count", ctx.default_desired_count), service_name = service_name, security_group = security_group ) scaling = service.auto_scale_task_count( max_capacity = ctx_srv.scaling.max_capacity, min_capacity = ctx_srv.scaling.min_capacity ) scaling.scale_on_cpu_utilization( id = "cpu_scaling", target_utilization_percent = ctx_srv.scaling.target_utilization_percent, scale_in_cooldown = core.Duration.seconds(ctx_srv.scaling.scale_in_cooldown_seconds), scale_out_cooldown = core.Duration.seconds(ctx_srv.scaling.scale_out_cooldown_seconds), )
def __init__( self, scope: core.Construct, id: str, vpc: ec2.IVpc, kinesis_stream: kinesis.IStream, **kwargs, ) -> None: super().__init__(scope, id, **kwargs) self._ecs_task_role = iam.Role( self, "ecsTaskRole", assumed_by=iam.ServicePrincipal("ecs-tasks.amazonaws.com"), description= "ECS Task Definition role to publish to Kinesis Data Stream", ) api_credentials = secretsmanager.Secret.from_secret_complete_arn( self, "apiCredentials", secret_complete_arn=API_CREDENTIALS_ARN, ) ecs_cluster = ecs.Cluster(self, "ecsCluster", vpc=vpc) ecs_task_definition = ecs.FargateTaskDefinition( self, "fargateTaskDefinition", task_role=self._ecs_task_role, ) ecs.ContainerDefinition( self, "ecsContainer", image=ecs.ContainerImage.from_asset( directory="containers/crypto_app", ), secrets={ "CRYPTO_COMPARE_API_KEY": ecs.Secret.from_secrets_manager(api_credentials, "CRYPTO_COMPARE_API_KEY"), # "TWITTER_API_KEY": ecs.Secret.from_secrets_manager( # api_credentials, "TWITTER_API_KEY" # ), # "TWITTER_SECRET_API_KEY": ecs.Secret.from_secrets_manager( # api_credentials, "TWITTER_SECRET_API_KEY" # ), # "TWITTER_BEARER_TOKEN": ecs.Secret.from_secrets_manager( # api_credentials, "TWITTER_BEARER_TOKEN" # ), # "TWITTER_ACCESS_TOKEN": ecs.Secret.from_secrets_manager( # api_credentials, "TWITTER_ACCESS_TOKEN" # ), # "TWITTER_SECRET_ACCESS_TOKEN": ecs.Secret.from_secrets_manager( # api_credentials, "TWITTER_SECRET_ACCESS_TOKEN" # ), }, environment={"KINESIS_STREAM_NAME": kinesis_stream.stream_name}, task_definition=ecs_task_definition, logging=ecs.LogDriver.aws_logs(stream_prefix="ecs"), ) ecs.FargateService( self, "fargateService", task_definition=ecs_task_definition, cluster=ecs_cluster, ) kinesis_stream.grant_read_write(ecs_task_definition.task_role)
def __init__(self, scope: core.Construct, id: str, stack_name: str, task_definition_cpu: int, task_definition_memory_limit_mib: int, docker_image_name: str, container_port: int, desired_container_count: int, private_subnets: Sequence[aws_ec2.Subnet] = None, public_subnets: Sequence[aws_ec2.Subnet] = None, private_security_group: aws_ec2.SecurityGroup = None, public_security_group: aws_ec2.SecurityGroup = None, vpc: aws_ec2.Vpc = None, fargate_cluster: aws_ecs.Cluster = None, authorizer_lambda_arn: str = None, authorizer_lambda_role_arn: str = None, **kwargs): super().__init__(scope, id, **kwargs) # Role self.role = aws_iam.Role( self, 'Role', assumed_by=aws_iam.ServicePrincipal(service='ecs.amazonaws.com'), managed_policies=[ aws_iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name= 'service-role/AmazonECSTaskExecutionRolePolicy') ], inline_policies={ id: aws_iam.PolicyDocument(statements=[ aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=[ 'kms:Encrypt', 'kms:Decrypt', 'kms:ReEncrypt*', 'kms:GenerateDataKey*', 'kms:DescribeKey', 'ec2:CreateNetworkInterface', 'ec2:DescribeNetworkInterfaces', 'ec2:DeleteNetworkInterface', # Remaining actions from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/quickref-ecs.html 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer', 'elasticloadbalancing:DeregisterTargets', 'elasticloadbalancing:Describe*', 'elasticloadbalancing:RegisterInstancesWithLoadBalancer', 'elasticloadbalancing:RegisterTargets', 'ec2:Describe*', 'ec2:AuthorizeSecurityGroupIngress' ], resources=['*']) ]) }) self.role.assume_role_policy.add_statements( aws_iam.PolicyStatement( actions=['sts:AssumeRole'], principals=[ aws_iam.ServicePrincipal(service='ecs-tasks.amazonaws.com') ])) # Set Defaults if parameters are None if vpc is None: vpc = aws_ec2.Vpc(self, 'Vpc') if private_subnets is None: private_subnets = vpc.private_subnets if public_subnets is None: public_subnets = vpc.public_subnets if public_security_group is None: public_security_group = aws_ec2.SecurityGroup( self, 'PublicSecurityGroup', vpc=vpc, allow_all_outbound=True) # Allow inbound HTTP traffic public_security_group.add_ingress_rule( peer=aws_ec2.Peer.ipv4(cidr_ip='0.0.0.0/0'), connection=aws_ec2.Port.tcp(port=80)) # Allow inbound HTTPS traffic public_security_group.add_ingress_rule( peer=aws_ec2.Peer.ipv4(cidr_ip='0.0.0.0/0'), connection=aws_ec2.Port.tcp(port=443)) if private_security_group is None: private_security_group = aws_ec2.SecurityGroup( self, 'PrivateSecurityGroup', vpc=vpc, allow_all_outbound=True) public_subnet_cidr_blocks = Utils.get_subnet_cidr_blocks( public_subnets) # Create an ingress rule for each of the NLB's subnet's CIDR ranges and add the rules to the ECS service's # security group. This will allow requests from the NLB to go into the ECS service. This allow inbound # traffic from public subnets. for cidr_block in public_subnet_cidr_blocks: private_security_group.add_ingress_rule( peer=aws_ec2.Peer.ipv4(cidr_ip=cidr_block), connection=aws_ec2.Port.tcp(port=container_port)) if fargate_cluster is None: fargate_cluster = aws_ecs.Cluster( self, 'FargateCluster', ) task_def = aws_ecs.FargateTaskDefinition( self, 'TaskDefinition', cpu=task_definition_cpu, memory_limit_mib=task_definition_memory_limit_mib, task_role=self.role, execution_role=self.role) container = aws_ecs.ContainerDefinition( self, 'Container', image=aws_ecs.ContainerImage.from_registry(name=docker_image_name), task_definition=task_def, logging=aws_ecs.AwsLogDriver(stream_prefix='/ecs')) container.add_port_mappings( aws_ecs.PortMapping(container_port=container_port, protocol=aws_ec2.Protocol.TCP)) ecs_service = aws_ecs.FargateService( self, 'FargateService', cluster=fargate_cluster, task_definition=task_def, vpc_subnets=aws_ec2.SubnetSelection(subnets=private_subnets), security_group=private_security_group, desired_count=desired_container_count) target_group = aws_elasticloadbalancingv2.NetworkTargetGroup( self, 'TargetGroup', port=80, # Health check occurs over HTTP health_check=aws_elasticloadbalancingv2.HealthCheck( protocol=aws_elasticloadbalancingv2.Protocol.TCP), targets=[ecs_service], vpc=vpc) nlb = aws_elasticloadbalancingv2.NetworkLoadBalancer( self, 'NetworkLoadBalancer', vpc=vpc, internet_facing=False, vpc_subnets=aws_ec2.SubnetSelection(subnets=public_subnets), ) nlb.add_listener( id='Listener', port=80, # HTTP listener default_target_groups=[target_group]) # nlb.log_access_logs( # todo: add this later when you have time to research the correct bucket policy. # bucket=aws_s3.Bucket( # self, 'LoadBalancerLogBucket', # bucket_name='load-balancer-logs', # public_read_access=False, # block_public_access=aws_s3.BlockPublicAccess( # block_public_policy=True, # restrict_public_buckets=True # ) # ) # ) # Dependencies ecs_service.node.add_dependency(nlb) # API Gateway rest_api = aws_apigateway.RestApi(self, stack_name) resource = rest_api.root.add_resource( path_part='{proxy+}', default_method_options=aws_apigateway.MethodOptions( request_parameters={'method.request.path.proxy': True})) token_authorizer = None if authorizer_lambda_arn and authorizer_lambda_role_arn: token_authorizer = aws_apigateway.TokenAuthorizer( #todo: make this a parameter? self, 'JwtTokenAuthorizer', results_cache_ttl=core.Duration.minutes(5), identity_source='method.request.header.Authorization', assume_role=aws_iam.Role.from_role_arn( self, 'AuthorizerLambdaInvokationRole', role_arn=authorizer_lambda_role_arn), handler=aws_lambda.Function.from_function_arn( self, 'AuthorizerLambda', function_arn=authorizer_lambda_arn)) resource.add_method( http_method='ANY', authorization_type=aws_apigateway.AuthorizationType.CUSTOM, authorizer=token_authorizer, integration=aws_apigateway.HttpIntegration( url=f'http://{nlb.load_balancer_dns_name}/{{proxy}}', http_method='ANY', proxy=True, options=aws_apigateway.IntegrationOptions( request_parameters={ 'integration.request.path.proxy': 'method.request.path.proxy' }, connection_type=aws_apigateway.ConnectionType.VPC_LINK, vpc_link=aws_apigateway.VpcLink( self, 'VpcLink', description= f'API Gateway VPC Link to internal NLB for {stack_name}', vpc_link_name=stack_name, targets=[nlb]))))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # EC2 Vpc construct vpc = ec2.Vpc( self, id="cromwell_server_vpc", max_azs=2 ) # ECS Cluster construct cluster = ecs.Cluster( self, id="cromwell_cluster", vpc=vpc ) # IAM roles ecstaskexecutionrole = iam.Role.from_role_arn( self, "ecstaskexecutionrole", role_arn="arn:aws:iam::562965587442:role/ecsTaskExecutionRole" ) batch_service_role = iam.Role.from_role_arn( self, "batchservicerole", role_arn="arn:aws:iam::562965587442:role/AWSBatchServiceRole" ) fargate_cromwell_role = iam.Role.from_role_arn( self, "fargate_cromwell_role", role_arn="arn:aws:iam::562965587442:role/fargate_cromwell_role" ) # Cromwell docker image from ECR container_img = ecr.Repository.from_repository_name( self, "cromwell_docker_image", repository_name=CROMWELL_REPOSITORY_NAME ) # ECS task definition construct task_def = ecs.TaskDefinition( self, "cromwell_server_task", execution_role=ecstaskexecutionrole, task_role=fargate_cromwell_role, compatibility=ecs.Compatibility.FARGATE, cpu="1024", memory_mib="4096" ) # ECS container definition construct container_def = ecs.ContainerDefinition( self, "cromwell_container", task_definition=task_def, image=ecs.ContainerImage.from_ecr_repository( repository=container_img, tag=CROMWELL_IMAGE_TAG ), command=["bash", "run_cromwell_server.sh"], cpu=1, health_check=None, working_directory='/', logging=ecs.LogDriver.aws_logs( stream_prefix="cromwell_logs", datetime_format=None, log_group=None, log_retention=None, multiline_pattern=None ) ) container_def.add_port_mappings( ecs.PortMapping( container_port=CROMWELL_PORT_NUMBER, host_port=CROMWELL_PORT_NUMBER, protocol=ecs.Protocol.TCP ) ) # EC2 Security Group construct security_group = ec2.SecurityGroup( self, "cromwell_server_security_group", vpc=vpc, allow_all_outbound=True, security_group_name="cromwell_server_security_group", description="This is the security group assigned to the cromwell server running as a Fargate service.", ) security_group.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port(protocol=ecs.Protocol.TCP, from_port=CROMWELL_PORT_NUMBER, to_port=CROMWELL_PORT_NUMBER, string_representation="cromwell_server_port"), remote_rule=None ) # ECS Fargate Service construct service = ecs.FargateService( self, "cromwell_service", task_definition=task_def, cluster=cluster, service_name="cromwell_server_service", assign_public_ip=True, desired_count=1, security_group=security_group ) # Batch resources # Reference: # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-batch-computeenvironment-computeresources.html # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-launchtemplatedata.html # with open("lib/aws_batch_launch_template_user_data.txt", 'r') as user_data_file: # user_data = user_data_file.read() # # ec2_user_data = ec2.UserData.custom(content=user_data) # ec2_instance =ec2.Instance( # self, # "ec2Instance", # instance_type=ec2.InstanceType("t2.small"), # machine_image=ec2.AmazonLinuxImage(), # vpc=vpc, # user_data=ec2_user_data # ) # launch_template_data = core.CfnResource( # self, # "cromwell_launch_template_data", # type="AWS::EC2::LaunchTemplate.LaunchTemplateData", # properties={ # "UserData": user_data # } # ) # # launch_template = ec2.CfnLaunchTemplate( # self, # "cromwell_launch_template", # launch_template_name="cromwell_launch_template", # launch_template_data=launch_template_data, # ) # # compute_resources = core.CfnResource( # self, # "cromwell_compute_resources", # type="AWS::Batch::ComputeEnvironment.ComputeResources", # properties={ # "DesiredvCpus": 256, # "Ec2KeyPair": "genovic-qc-eddev", # "InstanceRole": "arn:aws:iam::562965587442:role/ecsInstanceRole", # "InstanceTypes": ["optimal"], # "LaunchTemplate": launch_template.launch_template_name, # "MaxvCpus": 256, # "MinvCpus": 0, # "SecurityGroupIds": [vpc.vpc_default_security_group], # "Subnets": [subnet.subnet_id for subnet in vpc.public_subnets], # "Tags": "cromwell_compute_resource", # "Type": "EC2" # } # ) # # compute_env = batch.CfnComputeEnvironment( # self, # "cromwell_compute_env", # service_role=batch_service_role, # compute_environment_name="cromwell_compute_env", # type="MANAGED", # state="ENABLED", # compute_resources=compute_resources # ) # # queue = batch.CfnJobQueue( # self, # "cromwell_queue", # compute_environment_order=compute_env, # priority=1, # job_queue_name="cromwell_queue", # state="ENABLED" # ) # # core.CfnOutput( # self, # "cromwell_queue_name", # value=queue.job_queue_name # ) core.CfnOutput( self, "FargateCromwellServiceArn", value=service.service_arn )
def __init__(self, scope: cdk.Construct, id: str, name: str, vpc_name: str, security_group_name: str, secrets_path: str = "/ibc/paper/", trading_mode: str = "paper", **kwargs) -> None: super().__init__(scope, id, *kwargs) # TODO: Create Log Group # Create a cluster vpc = ec2.Vpc.from_lookup(self, "vpc", vpc_name=vpc_name) privateSubnets = vpc.private_subnets cluster = ecs.Cluster(self, "cluster", vpc=vpc) # TODO: check for namespace before adding below. This is failing on stack updates. cluster.add_default_cloud_map_namespace(name="private") task = ecs.FargateTaskDefinition(self, "task", cpu="512", memory_mi_b="1024") # Add SSM Permissions to IAM Role SSM_ACTIONS = ["ssm:GetParametersByPath", "kms:Decrypt"] SSM_RESOURCES = [ "arn:aws:kms:*:*:alias/aws/ssm", "arn:aws:ssm:*:*:parameter{}*".format(secrets_path), ] ssmPolicy = iam.PolicyStatement(iam.PolicyStatementEffect.Allow) for action in SSM_ACTIONS: ssmPolicy.add_action(action) for resource in SSM_RESOURCES: ssmPolicy.add_resource(resource) task.add_to_task_role_policy(ssmPolicy) ibcRepo = ecr.Repository.from_repository_name(self, "container_repo", "ibc") ibcImage = ecs.ContainerImage.from_ecr_repository(ibcRepo, "latest") # TODO: Add to Existing Hierarchal Logger, add log_group argument with ref to it ibcLogger = ecs.AwsLogDriver(self, "logger", stream_prefix=name) connectionLossMetric = logs.MetricFilter( self, "connectionLossMetric", filter_pattern=logs.FilterPattern.literal("ERROR ?110 ?130"), log_group=ibcLogger.log_group, metric_name="ib_connection_loss", metric_namespace=name, ) newContainerMetric = logs.MetricFilter( self, "newContainerMetric", filter_pattern=logs.FilterPattern.literal( "Starting virtual X frame buffer"), log_group=ibcLogger.log_group, metric_name="new_container", metric_namespace=name, ) kinesisFirehoseBucketActions = [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", ] kinesisFirehoseBucket = s3.Bucket(self, "firehoseBucket") kinesisFirehoseBucketPolicy = iam.PolicyStatement( iam.PolicyStatementEffect.Allow) for action in kinesisFirehoseBucketActions: kinesisFirehoseBucketPolicy.add_action(action) for resource in [ kinesisFirehoseBucket.bucket_arn, kinesisFirehoseBucket.bucket_arn + "/*", ]: kinesisFirehoseBucketPolicy.add_resource(resource) kinesisFirehoseBucketRole = iam.Role( self, "kinesisFirehoseBucketRole", assumed_by=iam.ServicePrincipal("firehose.amazonaws.com"), path="/service/" + name + "/", ) kinesisFirehoseBucketRole.add_to_policy(kinesisFirehoseBucketPolicy) kinesisFirehose = firehose.CfnDeliveryStream( self, "firehose", delivery_stream_name=name, delivery_stream_type="DirectPut", s3_destination_configuration={ "bucketArn": kinesisFirehoseBucket.bucket_arn, "bufferingHints": { "intervalInSeconds": 10 * 60, "sizeInMBs": 16 }, "compressionFormat": "GZIP", "roleArn": kinesisFirehoseBucketRole.role_arn, }, ) # Add Firehose Permissions to Task IAM Role FIREHOSE_ACTIONS = ["firehose:PutRecord", "firehose:PutRecordBatch"] firehosePolicy = iam.PolicyStatement(iam.PolicyStatementEffect.Allow) for action in FIREHOSE_ACTIONS: firehosePolicy.add_action(action) firehosePolicy.add_resource(kinesisFirehose.delivery_stream_arn) task.add_to_task_role_policy(firehosePolicy) environment = { "SECRETS_PATH": secrets_path, "TWS_LIVE_PAPER": trading_mode, "FIREHOSE_STREAM_NAME": kinesisFirehose.delivery_stream_name, } ibcContainer = ecs.ContainerDefinition( self, "container", task_definition=task, image=ibcImage, environment=environment, logging=ibcLogger, essential=True, ) securityGroup = ec2.SecurityGroup.from_security_group_id( self, "task_security_group", security_group_id=security_group_name) ibcService = ecs.FargateService( self, "fargate_service", cluster=cluster, task_definition=task, assign_public_ip=False, desired_count=1, security_group=securityGroup, service_discovery_options=ecs.ServiceDiscoveryOptions(name=name), service_name=name, vpc_subnets=privateSubnets, )
def configure_container(self, appname: str, props: Props, tgroups: {}): virtnodes = {} if appname == 'gateway' or appname == 'tcpecho': colors = [''] else: colors = props.colors for color in colors: fullname = color.upper()+appname td = ecs.FargateTaskDefinition(self, fullname+'_task', cpu='256', memory_mi_b='512', execution_role=props.taskexeciamrole, task_role=props.taskiamrole) env = {} if appname != 'tcpecho': td.node.find_child('Resource').add_property_override('proxyConfiguration', { 'type': 'APPMESH', 'containerName': 'envoy', 'proxyConfigurationProperties': [ {'name': 'IgnoredUID', 'value': '1337'}, {'name': 'ProxyIngressPort', 'value': '15000'}, {'name': 'ProxyEgressPort', 'value': '15001'}, {'name': 'AppPorts', 'value': '9080'}, {'name': 'EgressIgnoredIPs', 'value': '169.254.170.2,169.254.169.254'} ] }) env = { 'SERVER_PORT': '9080' } if appname != 'tcpecho': contimage = ecs.EcrImage.from_ecr_repository(props.repos[appname], tag='latest') else: contimage = ecs.ContainerImage.from_registry('cjimti/go-echo') port = 9080 if appname == 'gateway': env['COLOR_TELLER_ENDPOINT'] = props.repos['colorteller'].repository_name +\ '.'+props.cluster.default_namespace.namespace_name+':9080' env['TCP_ECHO_ENDPOINT'] = 'tcpecho.'+props.cluster.default_namespace.namespace_name+':2701' elif appname == 'colorteller': env['COLOR'] = color else: env = {'TCP_PORT': '2701', 'NODE_NAME': 'mesh/' + props.mesh.mesh_name + '/virtualNode/tcpecho--vn'} port = 2701 cont = ecs.ContainerDefinition(self, fullname+'-container', task_definition=td, essential=True, logging=ecs.AwsLogDriver(self, fullname+'-logs', stream_prefix=fullname), image=contimage, environment=env) #cont.add_port_mappings(container_port=port, host_port=port, protocol=ecs.Protocol.Tcp) # X-Ray and Envoy definition ---------------------------------------------------------------------------- if appname != 'tcpecho': xrayimage = ecs.ContainerImage.from_registry('amazon/aws-xray-daemon') xtask = td.add_container('xray-daemon', image=xrayimage, cpu=32, memory_reservation_mi_b=256, logging=ecs.AwsLogDriver(self, fullname+'-xray-logs', stream_prefix=fullname+'-xray'), essential=True, user='******') xtask.add_port_mappings(container_port=2000, host_port=2000, protocol=ecs.Protocol.Udp) # Envoy definition ---------------------------------------------------------------------------------- ENVOY_IMAGE_LOC = '111345817488.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.9.1.0-prod' envoyimage = ecs.EcrImage.from_registry(ENVOY_IMAGE_LOC) envoyenv = { 'APPMESH_VIRTUAL_NODE_NAME': 'mesh/'+props.mesh.mesh_name+'/virtualNode/'+appname+'-'+color+'-vn', 'ENABLE_ENVOY_XRAY_TRACING': '1', 'ENABLE_ENVOY_STATS_TAGS': '1', 'ENVOY_LOG_LEVEL': 'debug' } if appname == 'gateway': envoyenv['APPMESH_VIRTUAL_NODE_NAME'] = 'mesh/'+props.mesh.mesh_name+'/virtualNode/gateway--vn' envoy_hc = ecs.HealthCheck() envoy_hc['command'] = ['CMD-SHELL', 'curl -s http://localhost:9901/server_info | grep state | grep -q LIVE'] envoy_hc['interval'] = 5 envoy_hc['timeout'] = 2 envoy_hc['retries'] = 3 etask = td.add_container('envoy', image=envoyimage, user='******', essential=True, environment=envoyenv, logging=ecs.AwsLogDriver(self, fullname+'-envoy-logs', stream_prefix=fullname+'-envoy'), health_check=envoy_hc) etask.add_port_mappings(container_port=9901, host_port=9901, protocol=ecs.Protocol.Tcp) etask.add_port_mappings(container_port=15000, host_port=15000, protocol=ecs.Protocol.Tcp) etask.add_port_mappings(container_port=15001, host_port=15001, protocol=ecs.Protocol.Tcp) # Prometheus & Grafana definition for Gateway --------------------------------------------------------- if appname == 'gateway': prometheusimage = ecs.EcrImage.from_ecr_repository(props.repos['prometheus'], tag='latest') ptask = td.add_container('prometheus', image=prometheusimage, essential=True, logging=ecs.AwsLogDriver(self, appname + '-prometheus-logs', stream_prefix=appname + '-prometheus')) ptask.add_port_mappings(container_port=9090, host_port=9090) grafanaimage = ecs.ContainerImage.from_registry('grafana/grafana:latest') gtask = td.add_container('grafana', image=grafanaimage, essential=True, logging=ecs.AwsLogDriver(self, appname + '-grafana-logs', stream_prefix=appname + '-grafana')) gtask.add_port_mappings(container_port=3000, host_port=3000) disco = ecs.ServiceDiscoveryOptions() disco['dnsRecordType'] = sdisc.DnsRecordType.A disco['dnsTtlSec'] = 3000 if color == 'white': disco['name'] = 'colorteller' elif appname != 'gateway' and appname != 'tcpecho': disco['name'] = 'colorteller-'+color elif appname == 'gateway': disco['name'] = 'colorgateway' else: disco['name'] = 'tcpecho' svc = ecs.FargateService(self, fullname+'Service', maximum_percent=200, minimum_healthy_percent=100, desired_count=1, task_definition=td, cluster=props.cluster, vpc_subnets=props.vpc.private_subnets, security_group=props.csg, service_discovery_options=disco) if appname == 'gateway': svc._load_balancers = [{'containerName': 'grafana', 'containerPort': 3000, 'targetGroupArn': tgroups['grafana'].target_group_arn}] path = '/ping' if appname != 'tcpecho' else '/' spec = { 'listeners': [{ 'portMapping': {'port': port, 'protocol': 'http'}, 'healthCheck': {'protocol': 'http', 'path': path, 'healthyThreshold': 2, 'unhealthyThreshold': 2, 'timeoutMillis': 2000, 'intervalMillis': 5000}}], 'serviceDiscovery': { 'dns': {'hostname': svc._cloudmap_service.service_name+'.'+ props.cluster.default_namespace.namespace_name} } } if appname == 'gateway': spec['backends'] = [ {'virtualService': {'virtualServiceName': 'colorteller'+'.'+props.cluster.default_namespace.namespace_name}}, {'virtualService': {'virtualServiceName': 'tcpecho' + '.' + props.cluster.default_namespace.namespace_name}}, ] # Create AppMesh virtual nodes ------------------------------------------------------------------------ vn = appmesh.CfnVirtualNode(self, fullname + 'VirtualNode', mesh_name=props.mesh.mesh_name, virtual_node_name=appname + '-' + color + '-vn', spec=spec) virtnodes[fullname] = vn return virtnodes
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # The code that defines your stack goes here ############################################# #Import resorce and custom setting part start ############################################# #cn-north-1 impRes={ "vpc":"vpc-0883083ff3a10c1ec", "SvcSG":"sg-04d3b60e954c1c1ef", "ALBSG":"sg-0b6d093d52d48bba9", "ALBInternet":True, "taskRole":"arn:aws-cn:iam::627484392488:role/ecsTaskExecutionRole", "AlbSubnet":[ {"subnetId":"subnet-0d16fa0c969f234d3", "routeTabId":"rtb-074c6b532f3030ad6"}, {"subnetId":"subnet-0f28a97c04d3b11cd", "routeTabId":"rtb-074c6b532f3030ad6"} ], #"SvcSubNet":[{"subnetId":"subnet-0d16fa0c969f234d3","routeTabId":"rtb-074c6b532f3030ad6"}] "SvcSubNet":[{"subnetId":"subnet-0f28a97c04d3b11cd","routeTabId":"rtb-0587cc522717461cd"}, {"subnetId":"subnet-0d16fa0c969f234d3","routeTabId":"rtb-0587cc522717461cd"}] } newRes={ "TG":{"HealthPath":"/test.html","Port":80,"containPort":80}, "Listener":{"Port":80}, "TaskFamily":"tsFargate", "ImageAsset1":{"DockfilePath":"httpd-ssh", "BuildArgs":{"HTTP_PROXY":"http://YOUR_PROXY_SERVER:80"} } } MyTaskDefinition=[{"Cpu":512,"MemLimitMib":1024}] MyContainerDefinition=[ {"containerName":"MyContainer1", "cpu":256, "essential":True, "portMappings":[ecs.PortMapping(container_port=80,host_port=80)], #"portMappings":[ecs.PortMapping(container_port=80,host_port=80),ecs.PortMapping(container_port=22,host_port=22)], "environment":{"SSH_PUBLIC_KEY":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC/alWrS+HH5KkPbso+Tsy+Z0WGTX5wvXvon5OacLMyOU3gj2mbbIifasXf/RadpuywuyW3uFirtRlPmSb5Q0PVLODku503Xettw+u6/Z22VV7F2ACgg4iHaCo2SR4L8saUrLLfcKXKr/WCn3w7uYcqGsXEcSFCCSZgn4BoZJqP4Q=="}, "LogMountPoint":["/usr/local/apache2/logs"] } ] MySvc={"AssignPubIp":True, "desiredCount":1} ############################################# #Import resorce and custom setting part end ############################################# #if you import external resource app you cannot set destory policy #import VPC, Private Subnet, SG vpc = ec2.Vpc.from_lookup(self, "vpc", vpc_id=impRes["vpc"]) #import SG mysvcsg = ec2.SecurityGroup.from_security_group_id(self, "svcsg", impRes["SvcSG"], mutable=False) #import Role taskRole = iam.Role.from_role_arn(self, "TaskRole",impRes["taskRole"]) #create ALB mytargetGrp = elbv2.ApplicationTargetGroup(self, "targetGrp", target_type=elbv2.TargetType.IP, port=newRes["TG"]["Port"], vpc=vpc, health_check=elbv2.HealthCheck(path=newRes["TG"]["HealthPath"])) #target group cannot use .apply_removal_policy directly cfn_mytargetGrp=mytargetGrp.node.find_child("Resource") cfn_mytargetGrp.apply_removal_policy(cdk.RemovalPolicy.DESTROY) #import public subnet for alb albsubnets = [ ec2.Subnet.from_subnet_attributes(self,'albsubnetid1', subnet_id = impRes["AlbSubnet"][0]["subnetId"], route_table_id=impRes["AlbSubnet"][0]["routeTabId"] ), ec2.Subnet.from_subnet_attributes(self,'albsubnetid2', subnet_id = impRes["AlbSubnet"][1]["subnetId"], route_table_id=impRes["AlbSubnet"][1]["routeTabId"] ) ] vpc_subnets_selection = ec2.SubnetSelection(subnets=albsubnets) #create new ALB myalb = elbv2.ApplicationLoadBalancer(self, "ALBv2", vpc=vpc, security_group=ec2.SecurityGroup.from_security_group_id(self, "ALBSG", impRes["ALBSG"],mutable=False), internet_facing=impRes["ALBInternet"], vpc_subnets=vpc_subnets_selection) myalb.apply_removal_policy(cdk.RemovalPolicy.DESTROY) #create new ALB listener myalblistener = elbv2.ApplicationListener(self, "ALBlistenter", load_balancer=myalb, port=newRes["Listener"]["Port"]) myalblistener.apply_removal_policy(cdk.RemovalPolicy.DESTROY) myalblistener.add_target_groups("albaddtg", target_groups=[mytargetGrp]) #create new ECS Cluster mycluster = ecs.Cluster(self, "cluster", vpc=vpc) mycluster.apply_removal_policy(cdk.RemovalPolicy.DESTROY) fargatetaskDefinition = ecs.FargateTaskDefinition(self, "fargatetaskDefinition", cpu=MyTaskDefinition[0]["Cpu"], memory_limit_mib=MyTaskDefinition[0]["MemLimitMib"], execution_role=taskRole, family=newRes["TaskFamily"], task_role=taskRole) #volumes=myEfsVols) fargatetaskDefinition.apply_removal_policy(cdk.RemovalPolicy.DESTROY) #defind docker image asset dirname = os.path.dirname(__file__) #for container 1 normally httpd #create Image assent image will generated locally then push to ecr asset1 = DockerImageAsset(self, "ImageAsset1", directory=os.path.join(dirname, "../..", newRes["ImageAsset1"]["DockfilePath"]), build_args=newRes["ImageAsset1"]["BuildArgs"] ) #create container definition for task definition MyContainer1def = ecs.ContainerDefinition(self, "MyContainer1def", task_definition=fargatetaskDefinition, linux_parameters=ecs.LinuxParameters(self,"LinuxPara1",init_process_enabled=True), image=ecs.ContainerImage.from_ecr_repository(asset1.repository, asset1.image_uri.rpartition(":")[-1]), container_name=MyContainerDefinition[0]["containerName"], essential=MyContainerDefinition[0]["essential"], port_mappings=MyContainerDefinition[0]["portMappings"], environment=MyContainerDefinition[0]["environment"] ) #import service private subnet mysvcprivateSNs = [ ec2.Subnet.from_subnet_attributes(self,'svcprivateSN1', subnet_id = impRes["SvcSubNet"][0]["subnetId"], route_table_id=impRes["SvcSubNet"][0]["routeTabId"]), ec2.Subnet.from_subnet_attributes(self,'svcprivateSN2', subnet_id = impRes["SvcSubNet"][1]["subnetId"], route_table_id=impRes["SvcSubNet"][1]["routeTabId"]) ] #create service myservice=ecs.FargateService(self,"service", task_definition=fargatetaskDefinition, assign_public_ip=MySvc["AssignPubIp"], platform_version=ecs.FargatePlatformVersion.VERSION1_4, vpc_subnets=ec2.SubnetSelection(subnets=mysvcprivateSNs), security_group=mysvcsg, cluster=mycluster, desired_count=MySvc["desiredCount"]) mytargetGrp.add_target(myservice.load_balancer_target(container_name="MyContainer1",container_port=newRes["TG"]["containPort"], protocol=ecs.Protocol.TCP))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) vpc = ec2.Vpc( self, "MyVpc", max_azs=2 ) cluster = ecs.Cluster( self, "EC2Cluster", vpc=vpc ) security_group = ec2.SecurityGroup( self, "SecurityGroup", vpc=vpc, allow_all_outbound=True, ) security_group.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port.all_tcp(), description="Allow all traffic" ) app_target_group = elbv2.ApplicationTargetGroup( self, "AppTargetGroup", port=http_port, vpc=vpc, target_type=elbv2.TargetType.IP, ) elastic_loadbalancer = elbv2.ApplicationLoadBalancer( self, "ALB", vpc=vpc, internet_facing=True, security_group=security_group, ) app_listener = elbv2.ApplicationListener( self, "AppListener", load_balancer=elastic_loadbalancer, port=http_port, default_target_groups=[app_target_group], ) task_definition = ecs.TaskDefinition( self, "TaskDefenition", compatibility=ecs.Compatibility.FARGATE, cpu=task_def_cpu, memory_mib=task_def_memory_mb, ) container_defenition = ecs.ContainerDefinition( self, "ContainerDefenition", image=ecs.ContainerImage.from_registry("vulnerables/web-dvwa"), task_definition=task_definition, logging=ecs.AwsLogDriver( stream_prefix="DemoContainerLogs", log_retention=logs.RetentionDays.ONE_DAY, ), ) container_defenition.add_port_mappings( ecs.PortMapping( container_port=http_port, ) ) fargate_service = ecs.FargateService( self, "FargateService", task_definition=task_definition, cluster=cluster, security_group=security_group, ) fargate_service.attach_to_application_target_group( target_group=app_target_group, ) core.CfnOutput( self, "LoadBalancerDNS", value=elastic_loadbalancer.load_balancer_dns_name )
def __create_nlb_service(self, service_name: str, ctx: object): ctx_srv = getattr(ctx.inbound.services.nlb, service_name) ecs_task_role = self.__create_default_task_role(service_name) log_driver = ecs.LogDriver.aws_logs(log_group=self.log_group, stream_prefix=service_name) # create a Fargate task definition task_definition = ecs.FargateTaskDefinition( scope=self, id=f"{service_name}_task_definition", cpu=ctx_srv.size.cpu, memory_limit_mib=ctx_srv.size.ram, execution_role=self.ecs_exec_role, task_role=ecs_task_role, ) # create a container definition and associate with the Fargate task container_vars = self.__get_container_vars(service_name, ctx, ctx_srv) container = ecs.ContainerDefinition( scope=self, id=f"{service_name}_container_definition", task_definition=task_definition, image=ecs.ContainerImage.from_ecr_repository( self.ecr_repository, "latest"), logging=log_driver, **container_vars) security_group = ec2.SecurityGroup(scope=self, id=f"{service_name}_sg", vpc=self.vpc) service = ecs.FargateService( scope=self, id=f"{service_name}_service", task_definition=task_definition, cluster=self.cluster, desired_count=getattr(ctx_srv, "desired_count", ctx.default_desired_count), service_name=service_name, security_group=security_group, health_check_grace_period=core.Duration.minutes(10)) # map ports on the container for port in ctx_srv.ports: container.add_port_mappings( ecs.PortMapping(container_port=port, host_port=port, protocol=ecs.Protocol.TCP)) # add a listener to network load balancer listener = self.load_balancer.add_listener( id=f"{service_name}_{port}", port=port) security_group.add_ingress_rule( ec2.Peer.ipv4(ctx.ingress_cidr), ec2.Port.tcp(port), f"Logstash ingress for {service_name}") target = (service).load_balancer_target( container_name=container.container_name, container_port=port) listener.add_targets(id=f"{service_name}_{port}_tg", port=port, targets=[target]) scaling = service.auto_scale_task_count( max_capacity=ctx_srv.scaling.max_capacity, min_capacity=ctx_srv.scaling.min_capacity) scaling.scale_on_cpu_utilization( id="cpu_scaling", target_utilization_percent=ctx_srv.scaling. target_utilization_percent, scale_in_cooldown=core.Duration.seconds( ctx_srv.scaling.scale_in_cooldown_seconds), scale_out_cooldown=core.Duration.seconds( ctx_srv.scaling.scale_out_cooldown_seconds))
def __create_cloudmap_service(self, service_name: str, ctx: object): ctx_srv = getattr(ctx.inbound.services.cloudmap, service_name) ecs_task_role = self.__create_default_task_role(service_name) log_driver = ecs.LogDriver.aws_logs(log_group=self.log_group, stream_prefix=service_name) # create a Fargate task definition task_definition = ecs.FargateTaskDefinition( scope=self, id=f"{service_name}_task_definition", cpu=ctx_srv.size.cpu, memory_limit_mib=ctx_srv.size.ram, execution_role=self.ecs_exec_role, task_role=ecs_task_role, ) # create a container definition and associate with the Fargate task container_vars = self.__get_container_vars(service_name, ctx, ctx_srv) container = ecs.ContainerDefinition( scope=self, id=f"{service_name}_container_definition", task_definition=task_definition, image=ecs.ContainerImage.from_ecr_repository( self.ecr_repository, "latest"), logging=log_driver, **container_vars) security_group = ec2.SecurityGroup(scope=self, id=f"{service_name}_sg", vpc=self.vpc) service = ecs.FargateService(scope=self, id=f"{service_name}_service", task_definition=task_definition, cluster=self.cluster, desired_count=getattr( ctx_srv, "desired_count", ctx.default_desired_count), service_name=service_name, security_group=security_group) for port in ctx_srv.ports: container.add_port_mappings( ecs.PortMapping(container_port=port, host_port=port, protocol=ecs.Protocol.TCP)) security_group.add_ingress_rule( ec2.Peer.ipv4(ctx.ingress_cidr), ec2.Port.tcp(port), f"Logstash ingress for {service_name}") for port in ctx_srv.udp_ports: container.add_port_mappings( ecs.PortMapping(container_port=port, host_port=port, protocol=ecs.Protocol.UDP)) security_group.add_ingress_rule( ec2.Peer.ipv4(ctx.ingress_cidr), ec2.Port.udp(port), f"Logstash ingress for {service_name}") scaling = service.auto_scale_task_count( max_capacity=ctx_srv.scaling.max_capacity, min_capacity=ctx_srv.scaling.min_capacity) scaling.scale_on_cpu_utilization( id="cpu_scaling", target_utilization_percent=ctx_srv.scaling. target_utilization_percent, scale_in_cooldown=core.Duration.seconds( ctx_srv.scaling.scale_in_cooldown_seconds), scale_out_cooldown=core.Duration.seconds( ctx_srv.scaling.scale_out_cooldown_seconds)) cloudmap = awssd.PublicDnsNamespace.from_public_dns_namespace_attributes( scope=self, id=f"cloudmap_namespace", **ctx.inbound.namespace_props.dict()) service.enable_cloud_map(cloud_map_namespace=cloudmap, dns_record_type=awssd.DnsRecordType("A"), dns_ttl=core.Duration.seconds(15))
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc, cluster: ecs.Cluster, repository: ecr.Repository, **kwargs) -> None: super().__init__(scope, id, **kwargs) namespace = servicediscovery.PrivateDnsNamespace( scope=self, id="PRIVATE-DNS", vpc=vpc, name="private", description="a private dns" ) sg = ec2.SecurityGroup( scope=self, id="SG", vpc=vpc, allow_all_outbound=True, description="open 9200 and 9300 ports", security_group_name="es-group" ) sg.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port.tcp(port=9200), ) sg.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port.tcp(port=9300), ) ##################################################### elastic_task_def = ecs.Ec2TaskDefinition( scope=self, id="ES-TASK-DEF", network_mode=ecs.NetworkMode.AWS_VPC, volumes=[ecs.Volume( name="esdata", host=ecs.Host(source_path="/usr/share/elasticsearch/data"), )], ) elastic = ecs.ContainerDefinition( scope=self, id=constants.ES_CONTAINER_NAME, start_timeout=core.Duration.seconds(amount=30), task_definition=elastic_task_def, memory_limit_mib=4500, essential=True, image=ecs.ContainerImage.from_ecr_repository( repository=repository, tag='latest'), environment={ "cluster.name": constants.ES_CLUSTER_NAME, "bootstrap.memory_lock": "true", # "discovery.zen.ping.unicast.hosts": "elasticsearch", "node.name": constants.ES_CONTAINER_NAME, "node.master": "true", "node.data": "true", "ES_JAVA_OPTS": "-Xms4g -Xmx4g", }, logging=ecs.AwsLogDriver( stream_prefix="ES", log_retention=logs.RetentionDays.ONE_DAY, ), ) elastic.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.NOFILE, hard_limit=65535, soft_limit=65535)) elastic.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.MEMLOCK, hard_limit=-1, soft_limit=-1)) elastic.add_port_mappings(ecs.PortMapping(container_port=9200)) elastic.add_port_mappings(ecs.PortMapping(container_port=9300)) elastic.add_mount_points(ecs.MountPoint( container_path="/usr/share/elasticsearch/data", source_volume="esdata", read_only=False, )) # elastic.add_volumes_from(ecs.VolumeFrom( # source_container="esdata", # read_only=False, # )) es_service = ecs.Ec2Service( scope=self, id="ES-SERVICE", cluster=cluster, task_definition=elastic_task_def, desired_count=1, service_name="ES", security_group=sg, ) es_lb = elbv2.ApplicationLoadBalancer( scope=self, id="ES-ELB", vpc=vpc, internet_facing=True, ) es_listener = es_lb.add_listener( id="ES-LISTENER", port=80, ) es_service.register_load_balancer_targets( ecs.EcsTarget( new_target_group_id="ES-GRP", container_name=elastic.container_name, listener=ecs.ListenerConfig.application_listener( listener=es_listener, protocol=elbv2.ApplicationProtocol.HTTP), )) service = es_service.enable_cloud_map( cloud_map_namespace=namespace, dns_record_type=servicediscovery.DnsRecordType.A, # dns_ttl=core.Duration.seconds(amount=30), failure_threshold=1, name="elastic", ) core.CfnOutput( scope=self, id="DNS-ES", value=es_lb.load_balancer_dns_name, ) ##################################################### node_task_def = ecs.Ec2TaskDefinition( scope=self, id="NODE-TASK-DEF", network_mode=ecs.NetworkMode.AWS_VPC, volumes=[ecs.Volume( name="esdata", host=ecs.Host(source_path="/usr/share/elasticsearch/data"), )], ) node = ecs.ContainerDefinition( scope=self, id=constants.ES_NODE_CONTAINER_NAME, start_timeout=core.Duration.seconds(amount=40), task_definition=node_task_def, memory_limit_mib=4500, essential=True, image=ecs.ContainerImage.from_ecr_repository( repository=repository, tag='latest'), environment={ "cluster.name": constants.ES_CLUSTER_NAME, "bootstrap.memory_lock": "true", "discovery.zen.ping.unicast.hosts": "elastic.private", "node.name": constants.ES_NODE_CONTAINER_NAME, "node.master": "false", "node.data": "true", "ES_JAVA_OPTS": "-Xms4g -Xmx4g", }, logging=ecs.LogDrivers.aws_logs( stream_prefix="NODE", log_retention=logs.RetentionDays.ONE_DAY, )) node.add_port_mappings(ecs.PortMapping(container_port=9200)) node.add_port_mappings(ecs.PortMapping(container_port=9300)) node.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.NOFILE, hard_limit=65536, soft_limit=65536)) node.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.MEMLOCK, hard_limit=-1, soft_limit=-1)) node.add_mount_points(ecs.MountPoint( container_path="/usr/share/elasticsearch/data", source_volume="esdata", read_only=False, )) node_service = ecs.Ec2Service( scope=self, id="ES-NODE-SERVICE", cluster=cluster, task_definition=node_task_def, desired_count=1, service_name="NODE", security_group=sg, ) node_lb = elbv2.ApplicationLoadBalancer( scope=self, id="NODE-ELB", vpc=vpc, internet_facing=True, ) node_listener = node_lb.add_listener( id="NODE-LISTENER", port=80, ) node_service.register_load_balancer_targets( ecs.EcsTarget( new_target_group_id="NODE-GRP", container_name=node.container_name, listener=ecs.ListenerConfig.application_listener( listener=node_listener, protocol=elbv2.ApplicationProtocol.HTTP), )) core.CfnOutput( scope=self, id="DNS-NODE", value=node_lb.load_balancer_dns_name, )
def __init__(self, scope: core.Construct, id: str, vpc: ec2.Vpc, cluster: ecs.Cluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) elastic_cluster_task_def = ecs.Ec2TaskDefinition( scope=self, id="ES-TASK-DEF", network_mode=ecs.NetworkMode.BRIDGE, ) elastic = ecs.ContainerDefinition( scope=self, id=constants.ES_CONTAINER_NAME, start_timeout=core.Duration.seconds(amount=30), task_definition=elastic_cluster_task_def, memory_limit_mib=4024, essential=True, image=ecs.ContainerImage.from_registry( name="docker.elastic.co/elasticsearch/elasticsearch:6.8.6"), environment={ "cluster.name": constants.ES_CLUSTER_NAME, "bootstrap.memory_lock": "true", # "discovery.zen.ping.unicast.hosts": "elasticsearch", "node.name": constants.ES_CONTAINER_NAME, "node.master": "true", "node.data": "true", "ES_JAVA_OPTS": "-Xms2g -Xmx2g", }, logging=ecs.AwsLogDriver( stream_prefix="ES", log_retention=logs.RetentionDays.ONE_DAY, ), ) elastic.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.NOFILE, hard_limit=65535, soft_limit=65535)) elastic.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.MEMLOCK, hard_limit=-1, soft_limit=-1)) elastic.add_port_mappings(ecs.PortMapping(container_port=9200)) elastic.add_port_mappings(ecs.PortMapping(container_port=9300)) ##################################################### node = ecs.ContainerDefinition( scope=self, id=constants.ES_NODE_CONTAINER_NAME, start_timeout=core.Duration.seconds(amount=40), task_definition=elastic_cluster_task_def, memory_limit_mib=4024, essential=True, image=ecs.ContainerImage.from_registry( name="docker.elastic.co/elasticsearch/elasticsearch:6.8.6"), environment={ "cluster.name": constants.ES_CLUSTER_NAME, "bootstrap.memory_lock": "true", "discovery.zen.ping.unicast.hosts": constants.ES_CONTAINER_NAME, "node.name": constants.ES_NODE_CONTAINER_NAME, "node.master": "false", "node.data": "true", "ES_JAVA_OPTS": "-Xms2g -Xmx2g", }, logging=ecs.LogDrivers.aws_logs( stream_prefix="NODE", log_retention=logs.RetentionDays.ONE_DAY, )) node.add_port_mappings(ecs.PortMapping(container_port=9200)) node.add_port_mappings(ecs.PortMapping(container_port=9300)) node.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.NOFILE, hard_limit=65536, soft_limit=65536)) node.add_ulimits(ecs.Ulimit( name=ecs.UlimitName.MEMLOCK, hard_limit=-1, soft_limit=-1)) node.add_link(container=elastic, alias=constants.ES_CONTAINER_NAME) ##################################################### ecs_service = ecs.Ec2Service( scope=self, id="ES-SERVICE", cluster=cluster, task_definition=elastic_cluster_task_def, desired_count=1, service_name=constants.ECS_ES_SERVICE, ) lb = elbv2.ApplicationLoadBalancer( scope=self, id="ELB", vpc=vpc, internet_facing=True, ) listener = lb.add_listener( id="LISTENER", port=80, ) ecs_service.register_load_balancer_targets( ecs.EcsTarget( new_target_group_id="TARGET-GRP", container_name=elastic.container_name, # container_port=9200, listener=ecs.ListenerConfig.application_listener( listener=listener, protocol=elbv2.ApplicationProtocol.HTTP), )) core.CfnOutput( scope=self, id="DNS-NAME", value=lb.load_balancer_dns_name, )
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) public_subnets = ec2.SubnetConfiguration( name="Public", subnet_type=ec2.SubnetType.PUBLIC, cidr_mask=24) private_subnets = ec2.SubnetConfiguration( name="Tier2", subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT, cidr_mask=24) # create VPC vpc = ec2.Vpc(self, 'AWS-Cookbook-VPC', cidr='10.10.0.0/22', subnet_configuration=[public_subnets, private_subnets]) fargate_service_security_group = ec2.SecurityGroup( self, 'fargate_service_security_group', description='Security Group for the Fargate Service', allow_all_outbound=True, vpc=vpc) # create ECS Cluster ecs_cluster = ecs.Cluster(self, 'AWS-Cookbook-EcsCluster', cluster_name='awscookbook207', vpc=vpc) FargateTask = ecs.FargateTaskDefinition( self, 'FargateTask', cpu=256, memory_limit_mib=512, ) ContainerDef = ecs.ContainerDefinition( self, 'ContainerDef', image=ecs.ContainerImage.from_registry("amazon/amazon-ecs-sample"), task_definition=FargateTask, ) ContainerDef.add_port_mappings(ecs.PortMapping(container_port=80)) ecs.FargateService( self, 'awscookbook207Service', cluster=ecs_cluster, task_definition=FargateTask, assign_public_ip=False, desired_count=1, enable_ecs_managed_tags=False, # health_check_grace_period=core.Duration.seconds(60), max_healthy_percent=100, min_healthy_percent=0, platform_version=ecs.FargatePlatformVersion('LATEST'), security_groups=[fargate_service_security_group], service_name='awscookbook207Service', vpc_subnets=ec2.SubnetSelection( one_per_az=False, subnet_type=ec2.SubnetType('PRIVATE_WITH_NAT'))) # outputs CfnOutput(self, 'VpcId', value=vpc.vpc_id) CfnOutput(self, 'EcsClusterName', value=ecs_cluster.cluster_name) public_subnets = vpc.select_subnets(subnet_type=ec2.SubnetType.PUBLIC) CfnOutput(self, 'VpcPublicSubnets', value=', '.join(map(str, public_subnets.subnet_ids))) CfnOutput(self, 'AppSgId', value=fargate_service_security_group.security_group_id) CfnOutput(self, 'ContainerIp', value=fargate_service_security_group.security_group_id)