def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.python_build = assets.DockerImageAsset( self, 'PythonBuild', directory=os.path.join(src_root_dir, 'python-build'), repository_name='finsurf-python-build') self.cdk_deploy = assets.DockerImageAsset( self, 'CdkDeployImage', directory=os.path.join(src_root_dir, 'cdk-deploy'), repository_name='finsurf-cdk-env')
def __init__(self, scope: core.Construct, id: str, vpc: aws_ec2.IVpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) e2e_cluster = aws_ecs.Cluster(self, 'e2e-cluster', vpc=vpc, cluster_name='e2e-cluster') e2e_image = aws_ecr_assets.DockerImageAsset(self, 'e2e-image', directory='test/e2e') e2e_task = aws_ecs.FargateTaskDefinition(self, 'e2e-task', family='e2e-task') e2e_task.add_container( 'e2e-test-kafka', image=aws_ecs.ContainerImage.from_docker_image_asset(e2e_image), logging=aws_ecs.AwsLogDriver(stream_prefix='e2e')) e2e_security_group = aws_ec2.SecurityGroup(self, 'e2e', vpc=vpc) self.e2e_security_group = e2e_security_group # expose it to give it access to kafka core.CfnOutput(self, "subnets", value=','.join([ subnet.subnet_id for subnet in vpc.private_subnets ])) core.CfnOutput(self, "securitygroup", value=e2e_security_group.security_group_id)
def __init__(self, scope: core.Construct, id: str, repository_name: str, directory: str, subnet_group_name: str, context: InfraContext, securityGroups: typing.Optional[typing.List[ ec2.SecurityGroup]] = None, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.repo = assets.DockerImageAsset(self, 'Repo', directory=os.path.join( src_root_dir, directory), repository_name=repository_name) self.function = lambda_.DockerImageFunction( self, 'ContainerFunction', code=lambda_.DockerImageCode.from_ecr( repository=self.repo.repository, tag=self.repo.image_uri.split(':')[-1] ), # lambda_.DockerImageCode.from_image_asset(directory=os.path.join(src_root_dir,directory)), description='Python container lambda function for ' + repository_name, timeout=core.Duration.minutes(1), tracing=lambda_.Tracing.ACTIVE, vpc=context.networking.vpc, vpc_subnets=ec2.SubnetSelection( subnet_group_name=subnet_group_name), security_groups=securityGroups)
def __init__(self,scope:core.Construct, id:str, infra:RtspBaseResourcesConstruct, subnet_group_name:str='Default', **kwargs) -> None: super().__init__(scope, id, **kwargs) self.container = ecs.ContainerImage.from_docker_image_asset( asset=ecr.DockerImageAsset(self,'RtspConnectorContainer', directory='src/rtsp/connector', file='Dockerfile')) self.task_role = iam.Role(self,'TaskRole', assumed_by=iam.ServicePrincipal(service='ecs-tasks'), role_name='ecs-video-producer-task@homenet-{}'.format(core.Stack.of(self).region), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name='AmazonRekognitionFullAccess') ], description='Role for VideoSubnet Tasks') self.execution_role = iam.Role(self,'ExecutionRole', assumed_by=iam.ServicePrincipal(service='ecs-tasks'), role_name='ecs-rtsp-cluster-execution-role@homenet-{}'.format(core.Stack.of(self).region), description='ECS Execution Role for '+ RtspBaseResourcesConstruct.__name__) self.cluster = ecs.Cluster(self,'RtspCluster', vpc=self.landing_zone.vpc, cluster_name='{}-rtsp-services'.format(infra.landing_zone.zone_name)) # Tag all cluster resources for auto-domain join. core.Tags.of(self.cluster).add('domain','virtual.world') self.autoscale_group = autoscale.AutoScalingGroup(self,'WinASG', security_group=infra.landing_zone.security_group, instance_type= ec2.InstanceType.of( instance_class= ec2.InstanceClass.BURSTABLE3, instance_size=ec2.InstanceSize.SMALL), machine_image= ec2.MachineImage.generic_windows(ami_map={ 'us-east-1':'ami-0f93c815788872c5d' }), vpc= infra.landing_zone.vpc, role= self.execution_role, allow_all_outbound=True, associate_public_ip_address=False, #auto_scaling_group_name='{}-Rtsp-Windows'.format(landing_zone.zone_name), min_capacity= min_capacity, max_capacity= max_capacity, rolling_update_configuration= autoscale.RollingUpdateConfiguration( min_instances_in_service=0), update_type= autoscale.UpdateType.REPLACING_UPDATE, vpc_subnets=ec2.SubnetSelection(subnet_group_name=subnet_group_name)) self.cluster.add_auto_scaling_group( auto_scaling_group= self.autoscale_group, can_containers_access_instance_role=True, task_drain_time= task_drain_time) # Enable management from Managed AD and SSM for policy in ['AmazonSSMDirectoryServiceAccess','AmazonSSMManagedInstanceCore']: self.autoscale_group.role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name=policy))
def __init__(self, scope: core.Construct, id: str, vpc, redis, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.ecr = _ecr.Repository(self, "ecrRepo") self.ecs_cluster = _ecs.Cluster(self, "ecsCluster", container_insights=True, vpc=vpc) self.task_definition = _ecs.FargateTaskDefinition(self, "taskDefinition", memory_limit_mib=512, cpu=256) self.docker_image = _ecr_assets.DockerImageAsset(self, "dockerImage", directory="./code") self.container = self.task_definition.add_container( "testContainer", image=_ecs.ContainerImage.from_docker_image_asset( self.docker_image), logging=_ecs.LogDriver.aws_logs(stream_prefix="containerlogs"), environment={ "STAGE": "dev", "REDIS_ENDPOINT": redis.attr_configuration_end_point_address }, ) self.container.add_port_mappings( _ecs.PortMapping(container_port=5000, protocol=_ecs.Protocol.TCP)) self.service = _ecs.FargateService( self, "fargateService", cluster=self.ecs_cluster, task_definition=self.task_definition, desired_count=3, vpc_subnets=_ec2.SubnetSelection(subnets=vpc.private_subnets), security_groups=[vpc.sg]) self.lb = _elbv2.ApplicationLoadBalancer(self, "alb", vpc=vpc, security_group=vpc.sg, internet_facing=True) listener = self.lb.add_listener("listener", port=80) self.target_group = listener.add_targets("fargateTarget", port=80, targets=[self.service]) core.CfnOutput(self, "albDnsName", value=self.lb.load_balancer_dns_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Get the hosted Zone and create a certificate for our domain hosted_zone = route53.HostedZone.from_hosted_zone_attributes( self, "HostedZone", hosted_zone_id=HOSTED_ZONE_ID, zone_name=HOSTED_ZONE_NAME, ) cert = certificatemanager.DnsValidatedCertificate( self, "Certificate", hosted_zone=hosted_zone, domain_name=APP_DNS_NAME) # Set up a new VPC vpc = ec2.Vpc(self, "med-qaid-vpc", max_azs=2) # Set up an ECS Cluster for fargate cluster = ecs.Cluster(self, "med-qaid-cluster", vpc=vpc) # Define the Docker Image for our container (the CDK will do the build and push for us!) docker_image = ecr_assets.DockerImageAsset( self, "med-qaid-app", directory=os.path.join(os.path.dirname(__file__), "..", "src"), ) # Define the fargate service + ALB fargate_service = ecs_patterns.ApplicationLoadBalancedFargateService( self, "FargateService", cluster=cluster, certificate=cert, domain_name=f"{APP_DNS_NAME}", domain_zone=hosted_zone, cpu=2048, memory_limit_mib=16384, task_image_options={ "image": ecs.ContainerImage.from_docker_image_asset(docker_image), "environment": { "PORT": "80", }, }, ) # Allow 10 seconds for in flight requests before termination, the default of 5 minutes is much too high. fargate_service.target_group.set_attribute( key="deregistration_delay.timeout_seconds", value="10")
def __init__(self, scope: core.Stack, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create VPC self.vpc = ec2.Vpc.from_lookup(self, "VPC", vpc_name='api-gateway/VPC') # Create ECS Cluster self.ecs_cluster = ecs.Cluster(self, "ECSCluster", vpc=self.vpc) # This high level construct will build a docker image, ecr repo and connect the ecs service to allow pull access self.container_image = ecr.DockerImageAsset(self, "Image", directory="./") # Task definition details to define the frontend service container self.task_def = ecs_patterns.NetworkLoadBalancedTaskImageOptions( image=ecs.ContainerImage.from_ecr_repository( repository=self.container_image.repository), container_port=80, enable_logging=True, environment={"GIT_HASH": "12345"}, ) # Create the frontend service self.python_service = ecs_patterns.NetworkLoadBalancedFargateService( self, "PythonService", cpu=256, memory_limit_mib=512, cluster=self.ecs_cluster, desired_count=1, task_image_options=self.task_def, public_load_balancer=False, ) self.python_service.service.connections.allow_from_any_ipv4( port_range=ec2.Port( protocol=ec2.Protocol.ALL, string_representation="All port 80", from_port=80, ), description="Allows traffic on port 80 from NLB") # Create VPC Link from API Gateway to NLB # TODO: Make api id dynamic self.rest_api = apigw.RestApi.from_rest_api_id( self, "APIGateway", rest_api_id="6znhu1vqp6") # TODO: Create stage variable for vpc links self.gateway_vpc_link = apigw.VpcLink( self, "VPCLink", description="VPC Link from API Gateway to ECS Python Service", targets=[self.python_service.load_balancer], vpc_link_name="ECS_VPC_LINK")
def __init__(self, scope: core.Construct, id: str, resources: FsiSharedResources, subnet_group_name: str = 'Default', **kwargs) -> None: super().__init__(scope, id, **kwargs) # Configure role... role = iam.Role(self, 'Role', assumed_by=iam.ServicePrincipal(service='lambda'), description='Ameritrade Secrets Rotation Lambda via ' + self.component_name, role_name='{}@homenet-{}.{}'.format( self.component_name, resources.landing_zone.zone_name, core.Stack.of(self).region), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name= 'service-role/AWSLambdaVPCAccessExecutionRole') ]) resources.tda_secret.grant_write(role) # Configure the lambda... self.repo = assets.DockerImageAsset( self, 'Repo', directory='src/fsi/secret-rotation') code = lambda_.DockerImageCode.from_ecr( repository=self.repo.repository, tag=self.repo.image_uri.split(':')[-1]) self.function = lambda_.DockerImageFunction( self, 'Function', code=code, role=role, function_name='HomeNet-Fsi{}-{}'.format( resources.landing_zone.zone_name, self.component_name), description='Python container function for ' + self.component_name, timeout=core.Duration.minutes(15), tracing=lambda_.Tracing.ACTIVE, vpc=resources.landing_zone.vpc, log_retention=logs.RetentionDays.TWO_WEEKS, memory_size=3 * 128, allow_all_outbound=True, vpc_subnets=ec2.SubnetSelection( subnet_group_name=subnet_group_name), security_groups=[resources.landing_zone.security_group], environment={}) resources.tda_secret.grant_write(self.function.role) resources.tda_secret.grant_read(self.function.role)
def __init__(self, scope: core.Construct, id: str,datalake:DataLakeLayer, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.__datalake=datalake self.add_devbox() self.add_reviewer() AnalyzerLambda(scope,'fdroid-scrape-repo', project_name='fdroid-scrape-repo', datalake=datalake) AnalyzerLambda(scope,'review-result-sanitizer', project_name='review-result-sanitizer', datalake=datalake) ecr.DockerImageAsset(self,'fdroid-scrape', directory=os.path.join(root_dir,'fdroid-scrape'), repository_name='fdroid-scrape') ecr.DockerImageAsset(self,'review-result-downloader', directory=os.path.join(root_dir,'review-result-downloader'), repository_name='review-result-downloader')
def __init__(self, scope: core.Construct, id: str, infra: RtspBaseResourcesConstruct, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.infra = infra self.repo = assets.DockerImageAsset(self, 'Repo', directory=self.source_directory, file='Dockerfile') code = lambda_.DockerImageCode.from_ecr( repository=self.repo.repository, tag=self.repo.image_uri.split(':')[-1]) role = iam.Role(self, 'Role', assumed_by=iam.ServicePrincipal(service='lambda'), description='RtspFunction for the ' + self.component_name + ' component.', role_name='{}@homenet-{}.{}'.format( self.component_name, infra.landing_zone.zone_name, core.Stack.of(self).region), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name= 'service-role/AWSLambdaVPCAccessExecutionRole') ]) self.function = lambda_.DockerImageFunction( self, 'Function', code=code, role=role, function_name='HomeNet-{}-{}'.format(infra.landing_zone.zone_name, self.component_name), description='Python container lambda function for ' + self.component_name, timeout=self.function_timeout, tracing=lambda_.Tracing.ACTIVE, vpc=infra.landing_zone.vpc, log_retention=RetentionDays.FIVE_DAYS, memory_size=128, allow_all_outbound=True, vpc_subnets=ec2.SubnetSelection( subnet_group_name=infra.subnet_group_name), security_groups=[infra.security_group], environment={ 'REGION': core.Stack.of(self).region, }) self.dlq = sqs.Queue(self, 'DeadLetterQueue', queue_name=self.function.function_name + "_dlq")
def __init__(self, app: core.Construct, id: str, **kwargs) -> None: super().__init__(app, id) iam_role = RoleConstruct( self, 'role', assumed_by=iam.ServicePrincipal(ServicePrincipals.ECS_TASKS), managed_policies=[ManagedPolicies.AMAZON_S3_FULL_ACCESS]).iam_role self.taskDefinition = ecs.FargateTaskDefinition(self, 'task_definition', task_role=iam_role) # build a new docker image and push to Elastic Container Registry self.docker_image_asset = ecr_assets.DockerImageAsset( self, 'image_asset', directory='../src', build_args={'SCRIPTPATH': kwargs.get('script_path')}) # get the image from the built docker image asset self.docker_image = ecs.ContainerImage.from_docker_image_asset( self.docker_image_asset) # add container to task definition self.taskDefinition.add_container( 'container', image=self.docker_image, logging=ecs.LogDriver.aws_logs(stream_prefix='fargatetask'), environment={'sns_topic': kwargs.get('sns_topic').topic_name}) kwargs.get('sns_topic').grant_publish(self.taskDefinition.task_role) rule = events.Rule( self, "rule", description="trigger for fargate task", enabled=True, schedule=kwargs.get('schedule'), targets=[ events_targets.EcsTask( cluster=kwargs.get('cluster'), taskDefinition=self.taskDefinition, securityGroup=kwargs.get('ecs_security_group'), ) ])
def __init__(self, scope: core.Stack, id: str, vpc, cluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.vpc = vpc self.cluster = cluster # Building a custom image for jenkins leader. self.container_image = ecr.DockerImageAsset( self, "JenkinsWorkerDockerImage", directory='./docker/worker/') # Security group to connect workers to leader self.worker_security_group = ec2.SecurityGroup( self, "WorkerSecurityGroup", vpc=self.vpc, description="Jenkins Worker access to Jenkins leader", ) # IAM execution role for the workers to pull from ECR and push to CloudWatch logs self.worker_execution_role = iam.Role( self, "WorkerExecutionRole", assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'), ) self.worker_execution_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'service-role/AmazonECSTaskExecutionRolePolicy')) # Task role for worker containers - add to this role for any aws resources that jenkins requires access to self.worker_task_role = iam.Role( self, "WorkerTaskRole", assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'), ) # Create log group for workers to log self.worker_logs_group = logs.LogGroup( self, "WorkerLogGroup", retention=logs.RetentionDays.ONE_DAY) # Create log stream for worker log group self.worker_log_stream = logs.LogStream( self, "WorkerLogStream", log_group=self.worker_logs_group)
def __init__(self,scope:core.Construct, id:str, datalake:DataLakeLayer, project_name:str, concurrency:int=5, **kwargs) ->None: super().__init__(scope,id,**kwargs) self.__datalake = datalake repo = ecr.DockerImageAsset(self,'Repo', directory=os.path.join(root_dir, project_name), repository_name=project_name) self.function = lambda_.DockerImageFunction(self,project_name+'-repo', code = lambda_.DockerImageCode.from_ecr( repository=repo.repository, tag=repo.image_uri.split(':')[-1]), # lambda_.DockerImageCode.from_image_asset(directory=os.path.join(src_root_dir,directory)), description='Python container lambda function for '+repo.repository.repository_name, timeout= core.Duration.minutes(15), memory_size=4096, tracing= lambda_.Tracing.ACTIVE, # Note: This throttles the AWS S3 batch job. # Downloading too fast will cause f-droid to disconnect the crawler reserved_concurrent_executions= concurrency, filesystem= lambda_.FileSystem.from_efs_access_point( ap= self.datalake.efs.add_access_point( project_name, path='/'+project_name, create_acl=efs.Acl(owner_gid="0", owner_uid="0", permissions="777")), mount_path='/mnt/efs' ), environment={ 'EFS_MOUNT':'/mnt/efs' }, vpc= self.datalake.vpc) for name in [ 'AmazonElasticFileSystemClientFullAccess', 'AWSXrayWriteOnlyAccess', 'AmazonS3FullAccess', 'AWSCodeCommitFullAccess', 'AmazonCodeGuruReviewerFullAccess' ]: self.function.role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name(name))
def __build_task(self, directory:str, repository_name:str, entry_point:typing.Optional[typing.List[builtins.str]] = None, env_vars:dict={}, log_group_name:str=None): if log_group_name is None: log_group_name = '/finsurf/'+repository_name task_definition = ecs.FargateTaskDefinition( self,'FargateTaskDefinition') self.tda_secret.grant_read(task_definition.task_role) image = ecs.ContainerImage.from_docker_image_asset( asset=assets.DockerImageAsset( self,'DockerAsset', directory=path.join(src_root_dir,directory), repository_name=repository_name)) self.log_group = logs.LogGroup( self,'LogGroup', log_group_name=log_group_name, removal_policy=core.RemovalPolicy.DESTROY, retention=logs.RetentionDays.TWO_WEEKS) env_vars.update(self.tda_env_vars) task_definition.add_container('DefaultContainer', image=image, entry_point=entry_point, logging= ecs.AwsLogDriver( log_group=self.log_group, stream_prefix=repository_name, ), environment=env_vars, essential=True) self.__task_definition = task_definition
def __init__(self, scope: core.Stack, id: str, cluster, vpc, worker, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.cluster = cluster self.vpc = vpc self.worker = worker # Building a custom image for jenkins master. self.container_image = ecr.DockerImageAsset( self, "JenkinsMasterDockerImage", directory='./docker/master/') if config['DEFAULT']['fargate_enabled'] == "yes" or not config[ 'DEFAULT']['ec2_enabled'] == "yes": # Task definition details to define the Jenkins master container self.jenkins_task = ecs_patterns.ApplicationLoadBalancedTaskImageOptions( # image=ecs.ContainerImage.from_ecr_repository(self.container_image.repository), image=ecs.ContainerImage.from_docker_image_asset( self.container_image), container_port=8080, enable_logging=True, environment={ # https://github.com/jenkinsci/docker/blob/master/README.md#passing-jvm-parameters 'JAVA_OPTS': '-Djenkins.install.runSetupWizard=false', # https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/README.md#getting-started 'CASC_JENKINS_CONFIG': '/config-as-code.yaml', 'network_stack': self.vpc.stack_name, 'cluster_stack': self.cluster.stack_name, 'worker_stack': self.worker.stack_name, 'cluster_arn': self.cluster.cluster.cluster_arn, 'aws_region': config['DEFAULT']['region'], 'jenkins_url': config['DEFAULT']['jenkins_url'], 'subnet_ids': ",".join( [x.subnet_id for x in self.vpc.vpc.private_subnets]), 'security_group_ids': self.worker.worker_security_group.security_group_id, 'execution_role_arn': self.worker.worker_execution_role.role_arn, 'task_role_arn': self.worker.worker_task_role.role_arn, 'worker_log_group': self.worker.worker_logs_group.log_group_name, 'worker_log_stream_prefix': self.worker.worker_log_stream.log_stream_name }, ) # Create the Jenkins master service self.jenkins_master_service_main = ecs_patterns.ApplicationLoadBalancedFargateService( self, "JenkinsMasterService", cpu=int(config['DEFAULT']['fargate_cpu']), memory_limit_mib=int( config['DEFAULT']['fargate_memory_limit_mib']), cluster=self.cluster.cluster, desired_count=1, enable_ecs_managed_tags=True, task_image_options=self.jenkins_task, cloud_map_options=ecs.CloudMapOptions( name="master", dns_record_type=sd.DnsRecordType('A'))) self.jenkins_master_service = self.jenkins_master_service_main.service self.jenkins_master_task = self.jenkins_master_service.task_definition if config['DEFAULT']['ec2_enabled'] == "yes": self.jenkins_load_balancer = elb.ApplicationLoadBalancer( self, "JenkinsMasterELB", vpc=self.vpc.vpc, internet_facing=True, ) self.listener = self.jenkins_load_balancer.add_listener("Listener", port=80) self.jenkins_master_task = ecs.Ec2TaskDefinition( self, "JenkinsMasterTaskDef", network_mode=ecs.NetworkMode.AWS_VPC, volumes=[ ecs.Volume(name="efs_mount", host=ecs.Host(source_path='/mnt/efs')) ], ) self.jenkins_master_task.add_container( "JenkinsMasterContainer", image=ecs.ContainerImage.from_ecr_repository( self.container_image.repository), cpu=int(config['DEFAULT']['ec2_cpu']), memory_limit_mib=int( config['DEFAULT']['ec2_memory_limit_mib']), environment={ # https://github.com/jenkinsci/docker/blob/master/README.md#passing-jvm-parameters 'JAVA_OPTS': '-Djenkins.install.runSetupWizard=false', 'CASC_JENKINS_CONFIG': '/config-as-code.yaml', 'network_stack': self.vpc.stack_name, 'cluster_stack': self.cluster.stack_name, 'worker_stack': self.worker.stack_name, 'cluster_arn': self.cluster.cluster.cluster_arn, 'aws_region': config['DEFAULT']['region'], 'jenkins_url': config['DEFAULT']['jenkins_url'], 'subnet_ids': ",".join( [x.subnet_id for x in self.vpc.vpc.private_subnets]), 'security_group_ids': self.worker.worker_security_group.security_group_id, 'execution_role_arn': self.worker.worker_execution_role.role_arn, 'task_role_arn': self.worker.worker_task_role.role_arn, 'worker_log_group': self.worker.worker_logs_group.log_group_name, 'worker_log_stream_prefix': self.worker.worker_log_stream.log_stream_name }, logging=ecs.LogDriver.aws_logs( stream_prefix="JenkinsMaster", log_retention=logs.RetentionDays.ONE_WEEK), ) self.jenkins_master_task.default_container.add_mount_points( ecs.MountPoint(container_path='/var/jenkins_home', source_volume="efs_mount", read_only=False)) self.jenkins_master_task.default_container.add_port_mappings( ecs.PortMapping(container_port=8080, host_port=8080)) self.jenkins_master_service = ecs.Ec2Service( self, "EC2MasterService", task_definition=self.jenkins_master_task, cloud_map_options=ecs.CloudMapOptions( name="master", dns_record_type=sd.DnsRecordType('A')), desired_count=1, min_healthy_percent=0, max_healthy_percent=100, enable_ecs_managed_tags=True, cluster=self.cluster.cluster, ) self.target_group = self.listener.add_targets( "JenkinsMasterTarget", port=80, targets=[ self.jenkins_master_service.load_balancer_target( container_name=self.jenkins_master_task. default_container.container_name, container_port=8080, ) ], deregistration_delay=core.Duration.seconds(10)) # Opening port 5000 for master <--> worker communications self.jenkins_master_service.task_definition.default_container.add_port_mappings( ecs.PortMapping(container_port=50000, host_port=50000)) # Enable connection between Master and Worker self.jenkins_master_service.connections.allow_from( other=self.worker.worker_security_group, port_range=ec2.Port(protocol=ec2.Protocol.TCP, string_representation='Master to Worker 50000', from_port=50000, to_port=50000)) # Enable connection between Master and Worker on 8080 self.jenkins_master_service.connections.allow_from( other=self.worker.worker_security_group, port_range=ec2.Port(protocol=ec2.Protocol.TCP, string_representation='Master to Worker 8080', from_port=8080, to_port=8080)) # IAM Statements to allow jenkins ecs plugin to talk to ECS as well as the Jenkins cluster # self.jenkins_master_task.add_to_task_role_policy( iam.PolicyStatement( actions=[ "ecs:RegisterTaskDefinition", "ecs:DeregisterTaskDefinition", "ecs:ListClusters", "ecs:DescribeContainerInstances", "ecs:ListTaskDefinitions", "ecs:DescribeTaskDefinition", "ecs:DescribeTasks" ], resources=["*"], )) self.jenkins_master_task.add_to_task_role_policy( iam.PolicyStatement(actions=["ecs:ListContainerInstances"], resources=[self.cluster.cluster.cluster_arn])) self.jenkins_master_task.add_to_task_role_policy( iam.PolicyStatement( actions=["ecs:RunTask"], resources=[ "arn:aws:ecs:{0}:{1}:task-definition/fargate-workers*". format( self.region, self.account, ) ])) self.jenkins_master_task.add_to_task_role_policy( iam.PolicyStatement(actions=["ecs:StopTask"], resources=[ "arn:aws:ecs:{0}:{1}:task/*".format( self.region, self.account) ], conditions={ "ForAnyValue:ArnEquals": { "ecs:cluster": self.cluster.cluster.cluster_arn } })) self.jenkins_master_task.add_to_task_role_policy( iam.PolicyStatement(actions=["iam:PassRole"], resources=[ self.worker.worker_task_role.role_arn, self.worker.worker_execution_role.role_arn ])) # END OF JENKINS ECS PLUGIN IAM POLICIES # self.jenkins_master_task.add_to_task_role_policy( iam.PolicyStatement( actions=["*"], resources=[self.worker.worker_logs_group.log_group_arn]))
def __init__(self, scope: core.Stack, id: str, base_module, stream_module, **kwargs): super().__init__(scope, id, **kwargs) self.base_module = base_module self.stream_module = stream_module # Added the keys manually, this needs to be imported. Likely should be an argument we pass to stack, ie self.secrets_arn = passed-in-to-constructor self.twitter_secrets = aws_secretsmanager.Secret(self, "TwitterSecrets").from_secret_arn( self, "TwitterSecretARN", secret_arn=self.node.try_get_context("TWITTER_SECRET_ARN") ) self.cluster = aws_ecs.Cluster( self, "ECSCluster", vpc=self.base_module.vpc, ) #Queue to push last updated id self.twitter_id_queue = aws_sqs.Queue( self, "TwitterWorkerQueue", queue_name="{}.fifo".format(self.stack_name), fifo=True, content_based_deduplication=True, visibility_timeout=core.Duration.seconds(90) ) # SSM parameter to indicate if initial run has occurred self.initial_run_parameter = aws_ssm.StringParameter( self, "StringParameterInitialRun", parameter_name=f"/{self.stack_name}-NOT-first-run", string_value='False', description="Parameter for twitter stream feed to set to true after first run has occurred and an object has been put in queue" ) self.task_definition = aws_ecs.FargateTaskDefinition( self, "TwitterWorkerTD", cpu=256, memory_limit_mib=512 ) self.task_definition.add_container( "ContainerImage", image=aws_ecs.ContainerImage.from_docker_image_asset( aws_ecr_assets.DockerImageAsset( self, "DockerImage", directory='./', exclude=["cdk.out"] ) ), logging=aws_ecs.AwsLogDriver(stream_prefix=self.stack_name, log_retention=aws_logs.RetentionDays.THREE_DAYS), environment={ "FIREHOSE_NAME": self.stream_module.firehose.delivery_stream_name, "SQS_QUEUE_NAME": self.twitter_id_queue.queue_name, "SSM_PARAM_INITIAL_RUN": self.initial_run_parameter.parameter_name, "TWITTER_KEYWORD": os.getenv("TWITTER_KEYWORD") or 'maga', "SINCE_DATE": '2019-03-01', "WORLD_ID": '23424977' }, secrets=[ aws_ecs.Secret.from_secrets_manager( secret=self.twitter_secrets ) ] ) # IAM Permissions for fargate task # Adding firehose put policy self.firehose_iam_policy_statement = aws_iam.PolicyStatement() self.firehose_iam_policy_statement.add_actions('firehose:Put*') self.firehose_iam_policy_statement.add_resources(self.stream_module.firehose.attr_arn) # Permission to talk to comprehend for sentiment analysis self.comprehend_iam_policy_statement = aws_iam.PolicyStatement() self.comprehend_iam_policy_statement.add_actions('comprehend:*') self.comprehend_iam_policy_statement.add_all_resources() self.task_iam_policy = aws_iam.Policy( self, "IAMPolicyTwitterStreamFargate", policy_name="TwitterStreamingFargate-{}".format(self.stack_name), statements=[self.firehose_iam_policy_statement], ) self.twitter_secrets.grant_read(self.task_definition.task_role) self.task_iam_policy.attach_to_role(self.task_definition.task_role) self.twitter_id_queue.grant_send_messages(self.task_definition.task_role) self.twitter_id_queue.grant_consume_messages(self.task_definition.task_role) self.initial_run_parameter.grant_read(self.task_definition.task_role) self.initial_run_parameter.grant_write(self.task_definition.task_role) self.fargate_service = aws_ecs.FargateService( self, "TwitterWorker", service_name=self.stack_name, task_definition=self.task_definition, cluster=self.cluster )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here # Create a VPC myvpc = ec2.Vpc(self, "CDKVPC", cidr=vars.cidr) # SG for ELB creation websitefrontendSG = ec2.SecurityGroup( self, 'websitefrontendSG', vpc=myvpc, security_group_name='websitefrontendSG') websitefrontendSG.add_ingress_rule(peer=ec2.Peer.ipv4('0.0.0.0/0'), connection=ec2.Port.tcp(80)) websitefrontendSG.add_ingress_rule(peer=ec2.Peer.ipv4('0.0.0.0/0'), connection=ec2.Port.tcp(443)) # Create ALB in VPC alb = elb.ApplicationLoadBalancer( self, 'websitefrontend-public', vpc=myvpc, load_balancer_name='websitefrontend-public', security_group=websitefrontendSG, internet_facing=True) # Add target group to ALB catalogtargetgroup = elb.ApplicationTargetGroup( self, 'CatalogTargetGroup', port=80, vpc=myvpc, target_type=elb.TargetType.IP) if not vars.sslcert: # Add http listener to ALB alblistenerhttp = elb.ApplicationListener( self, 'alblistenerhttp', load_balancer=alb, default_target_groups=[catalogtargetgroup], port=80) if vars.sslcert: # Add http listener to ALB alblistenerhttp = elb.ApplicationListener(self, 'alblistenerhttp', load_balancer=alb, port=80) elb.ApplicationListenerRule(self, 'httpredirectionrule', listener=alblistenerhttp, redirect_response=elb.RedirectResponse( status_code='HTTP_301', port='443', protocol='HTTPS')) # OPTIONAL - Add https listener to ALB & attach certificate alblistenerhttps = elb.ApplicationListener( self, 'alblistenerhttps', load_balancer=alb, default_target_groups=[catalogtargetgroup], port=443, certificate_arns=[vars.sslcert_arn]) # OPTIONAL - Redirect HTTP to HTTPS alblistenerhttp.add_redirect_response(id='redirectionrule', port='443', status_code='HTTP_301', protocol='HTTPS') if vars.customdomain: # OPTIONAL - Update DNS with ALB webshopxyz_zone = r53.HostedZone.from_hosted_zone_attributes( self, id='customdomain', hosted_zone_id=vars.hosted_zone_id, zone_name=vars.zone_name) webshop_root_record = r53.ARecord( self, 'ALBAliasRecord', zone=webshopxyz_zone, target=r53.RecordTarget.from_alias( alias.LoadBalancerTarget(alb))) # SG for ECS creation ECSSG = ec2.SecurityGroup(self, 'ECSSecurityGroup', vpc=myvpc, security_group_name='ECS') ECSSG.add_ingress_rule(peer=websitefrontendSG, connection=ec2.Port.tcp(80)) # SG for MySQL creation MySQLSG = ec2.SecurityGroup(self, 'DBSecurityGroup', vpc=myvpc, security_group_name='DB') MySQLSG.add_ingress_rule(peer=ECSSG, connection=ec2.Port.tcp(3306)) # Create DB subnet group subnetlist = [] for subnet in myvpc.private_subnets: subnetlist.append(subnet.subnet_id) subnetgr = rds.CfnDBSubnetGroup( self, 'democlustersubnetgroup', db_subnet_group_name='democlustersubnetgroup', db_subnet_group_description='DemoCluster', subnet_ids=subnetlist) # Create secret db passwd secret = sm.SecretStringGenerator( exclude_characters="\"'@/\\", secret_string_template='{"username": "******"}', generate_string_key='password', password_length=40) dbpass = sm.Secret(self, 'democlusterpass', secret_name='democlusterpass', generate_secret_string=secret) # Create Aurora serverless MySQL instance dbcluster = rds.CfnDBCluster( self, 'DemoCluster', engine='aurora', engine_mode='serverless', engine_version='5.6', db_cluster_identifier='DemoCluster', master_username=dbpass.secret_value_from_json( 'username').to_string(), master_user_password=dbpass.secret_value_from_json( 'password').to_string(), storage_encrypted=True, port=3306, vpc_security_group_ids=[MySQLSG.security_group_id], scaling_configuration=rds.CfnDBCluster. ScalingConfigurationProperty(auto_pause=True, max_capacity=4, min_capacity=1, seconds_until_auto_pause=300), db_subnet_group_name=subnetgr.db_subnet_group_name) dbcluster.add_override('DependsOn', 'democlustersubnetgroup') # Attach database to secret attach = sm.CfnSecretTargetAttachment( self, 'RDSAttachment', secret_id=dbpass.secret_arn, target_id=dbcluster.ref, target_type='AWS::RDS::DBCluster') # Upload image into ECR repo ecrdemoimage = ecra.DockerImageAsset(self, 'ecrdemoimage', directory='../', repository_name='demorepo', exclude=['cdk.out']) # Create ECS fargate cluster ecscluster = ecs.Cluster(self, "ecsCluster", vpc=myvpc) # Create task role for productsCatalogTask getsecretpolicystatement = iam.PolicyStatement(actions=[ "secretsmanager:GetResourcePolicy", "secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret", "secretsmanager:ListSecretVersionIds" ], resources=[ dbpass.secret_arn ], effect=iam.Effect.ALLOW) getsecretpolicydocument = iam.PolicyDocument( statements=[getsecretpolicystatement]) taskrole = iam.Role( self, 'TaskRole', assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com'), role_name='TaskRoleforproductsCatalogTask', inline_policies=[getsecretpolicydocument]) # Create task definition taskdefinition = ecs.FargateTaskDefinition(self, 'productsCatalogTask', cpu=1024, memory_limit_mib=2048, task_role=taskrole) # Add container to task definition productscatalogcontainer = taskdefinition.add_container( 'productscatalogcontainer', image=ecs.ContainerImage.from_docker_image_asset( asset=ecrdemoimage), environment={ "region": vars.region, "secretname": "democlusterpass" }) productscatalogcontainer.add_port_mappings( ecs.PortMapping(container_port=80, host_port=80)) # Create service and associate it with the cluster catalogservice = ecs.FargateService( self, 'catalogservice', task_definition=taskdefinition, assign_public_ip=False, security_group=ECSSG, vpc_subnets=ec2.SubnetSelection(subnets=myvpc.select_subnets( subnet_type=ec2.SubnetType.PRIVATE).subnets), cluster=ecscluster, desired_count=2) # Add autoscaling to the service scaling = catalogservice.auto_scale_task_count(max_capacity=20, min_capacity=1) scaling.scale_on_cpu_utilization( 'ScaleOnCPU', target_utilization_percent=70, scale_in_cooldown=core.Duration.seconds(amount=1), scale_out_cooldown=core.Duration.seconds(amount=0)) # Associate the fargate service with load balancer targetgroup catalogservice.attach_to_application_target_group(catalogtargetgroup)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) container_image = aws_ecr_assets.DockerImageAsset( self, 'dockerImage', directory='./unicorn_api_service') # noinspection PyTypeChecker unicorn_service = ecs_patterns.ApplicationLoadBalancedFargateService( self, 'unicornService', memory_limit_mib=1024, cpu=512, task_image_options={ 'image': ecs.ContainerImage.from_docker_image_asset( asset=container_image), 'container_port': 80, 'enable_logging': True, 'environment': { 'FLASK_DEBUG': '1', 'FLASK_ENV': 'development', 'FLASK_APP': '/app/app.py', 'PYTHONUNBUFFERED': '1' } }) unicorn_service.task_definition.execution_role.add_managed_policy( iam.ManagedPolicy.from_managed_policy_arn( self, 'ecsExecutionRole', managed_policy_arn= 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy' )) unicorn_service.target_group.configure_health_check(path="/health") unicorn_service.task_definition.task_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name( 'AmazonSSMManagedInstanceCore')) user_policy = iam.ManagedPolicy( self, 'userPolicy', document=iam.PolicyDocument(statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['ecs:UpdateService'], resources=[ 'arn:aws:ecs:*:*:service/*/cloud-debug-*', unicorn_service.service.service_arn ]), iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'iam:GetRole', 'iam:ListRoles', 'iam:SimulatePrincipalPolicy' ], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['iam:PassRole'], resources=[ unicorn_service.task_definition.execution_role. role_arn, unicorn_service.task_definition.task_role.role_arn ], conditions={ 'StringEquals': { "iam:PassedToService": "ecs-tasks.amazonaws.com" } }), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['iam:PassRole'], resources=[ 'arn:aws:iam::*:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS' ]), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 's3:CreateBucket', 's3:GetObject', 's3:PutObject', 's3:DeleteObject', 's3:ListBucket' ], resources=['arn:aws:s3:::do-not-delete-cloud-debug-*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 'ecs:ListClusters', 'ecs:ListServices', 'ecs:DescribeServices', 'ecs:ListTasks', 'ecs:DescribeTasks', 'ecs:DescribeTaskDefinition', 'elasticloadbalancing:DescribeListeners', 'elasticloadbalancing:DescribeRules', 'elasticloadbalancing:DescribeTargetGroups', 'ecr:GetAuthorizationToken', 'ecr:BatchCheckLayerAvailability', 'ecr:GetDownloadUrlForLayer', 'ecr:BatchGetImage' ], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['logs:CreateLogGroup', 'logs:CreateLogStream'], resources=['arn:aws:logs:*:*:cloud-debug*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['ecs:CreateService', 'ecs:DeleteService'], resources=['arn:aws:ecs:*:*:service/*/cloud-debug*']), iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=['ecs:RegisterTaskDefinition'], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 'elasticloadbalancing:ModifyListener', 'elasticloadbalancing:ModifyRule', 'elasticloadbalancing:ModifyTargetGroupAttributes' ], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 'elasticloadbalancing:CreateTargetGroup', 'elasticloadbalancing:DeleteTargetGroup' ], resources=[ 'arn:aws:elasticloadbalancing:*:*:targetgroup/cloud-debug*' ]), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 'ssm:StartSession', 'ssm:TerminateSession', 'ssm:ResumeSession', 'ssm:DescribeSessions', 'ssm:GetConnectionStatus' ], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ 'application-autoscaling:RegisterScalableTarget', 'application-autoscaling:DeregisterScalableTarget', 'application-autoscaling:DescribeScalableTargets' ], resources=['*']), ]))
def __init__(self, scope: core.Construct, id: str, *, vpc="", health_check_path="/", env_level="prd", **kwargs) -> None: super().__init__(scope, id, **kwargs) web_asset = ecr_assets.DockerImageAsset(self, 'web_asset', directory="docker", file="web/Dockerfile") app_asset = ecr_assets.DockerImageAsset(self, 'app_asset', directory="docker", file="app/Dockerfile") cluster = ecs.Cluster(self, "Cluster", vpc=vpc) alb_options = ecs_patterns.ApplicationLoadBalancedTaskImageOptions( image=ecs.ContainerImage.from_ecr_repository( web_asset.repository, web_asset.image_uri[-64:]), environment={ "ENV": env_level, }) service = ecs_patterns.ApplicationLoadBalancedFargateService( self, "fargate", cluster=cluster, desired_count=1, task_image_options=alb_options, public_load_balancer=True, listener_port=80) service.target_group.configure_health_check( path=health_check_path, healthy_threshold_count=2, healthy_http_codes="200-399", unhealthy_threshold_count=2, timeout=core.Duration.seconds(10), interval=core.Duration.seconds(15)) service.target_group.enable_cookie_stickiness(core.Duration.hours(1)) service.target_group.set_attribute( "deregistration_delay.timeout_seconds", "10") service.task_definition.add_container( "app", image=ecs.ContainerImage.from_ecr_repository( app_asset.repository, app_asset.image_uri[-64:]), logging=ecs.LogDriver.aws_logs(stream_prefix="fargate"), environment={ "ENV": env_level, }) scalable_target = service.service.auto_scale_task_count( max_capacity=20) scalable_target.scale_on_cpu_utilization( "CpuScaling", target_utilization_percent=50, scale_in_cooldown=core.Duration.seconds(60), scale_out_cooldown=core.Duration.seconds(10)) scalable_target.scale_on_memory_utilization( "MemoryScaling", target_utilization_percent=80, scale_in_cooldown=core.Duration.seconds(60), scale_out_cooldown=core.Duration.seconds(10))
def __init__(self, scope: core.Construct, construct_id: str, properties: WordpressStackProperties, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) database = rds.ServerlessCluster( self, "WordpressServerless", engine=rds.DatabaseClusterEngine.AURORA_MYSQL, default_database_name="WordpressDatabase", vpc=properties.vpc, scaling=rds.ServerlessScalingOptions( auto_pause=core.Duration.seconds(0)), deletion_protection=False, backup_retention=core.Duration.days(7), removal_policy=core.RemovalPolicy.DESTROY, ) file_system = efs.FileSystem( self, "WebRoot", vpc=properties.vpc, performance_mode=efs.PerformanceMode.GENERAL_PURPOSE, throughput_mode=efs.ThroughputMode.BURSTING, ) # docker context directory docker_context_path = os.path.dirname(__file__) + "../../src" # upload images to ecr nginx_image = ecr_assets.DockerImageAsset( self, "Nginx", directory=docker_context_path, file="Docker.nginx", ) wordpress_image = ecr_assets.DockerImageAsset( self, "Php", directory=docker_context_path, file="Docker.wordpress", ) cluster = ecs.Cluster(self, 'ComputeResourceProvider', vpc=properties.vpc) wordpress_volume = ecs.Volume( name="WebRoot", efs_volume_configuration=ecs.EfsVolumeConfiguration( file_system_id=file_system.file_system_id)) event_task = ecs.FargateTaskDefinition(self, "WordpressTask", volumes=[wordpress_volume]) # # webserver # nginx_container = event_task.add_container( "Nginx", image=ecs.ContainerImage.from_docker_image_asset(nginx_image)) nginx_container.add_port_mappings(ecs.PortMapping(container_port=80)) nginx_container_volume_mount_point = ecs.MountPoint( read_only=True, container_path="/var/www/html", source_volume=wordpress_volume.name) nginx_container.add_mount_points(nginx_container_volume_mount_point) # # application server # app_container = event_task.add_container( "Php", environment={ 'WORDPRESS_DB_HOST': database.cluster_endpoint.hostname, 'WORDPRESS_TABLE_PREFIX': 'wp_' }, secrets={ 'WORDPRESS_DB_USER': ecs.Secret.from_secrets_manager(database.secret, field="username"), 'WORDPRESS_DB_PASSWORD': ecs.Secret.from_secrets_manager(database.secret, field="password"), 'WORDPRESS_DB_NAME': ecs.Secret.from_secrets_manager(database.secret, field="dbname"), }, image=ecs.ContainerImage.from_docker_image_asset(wordpress_image)) app_container.add_port_mappings(ecs.PortMapping(container_port=9000)) container_volume_mount_point = ecs.MountPoint( read_only=False, container_path="/var/www/html", source_volume=wordpress_volume.name) app_container.add_mount_points(container_volume_mount_point) # # create service # wordpress_service = ecs.FargateService( self, "InternalService", task_definition=event_task, platform_version=ecs.FargatePlatformVersion.VERSION1_4, cluster=cluster, ) # # scaling # scaling = wordpress_service.auto_scale_task_count(min_capacity=2, max_capacity=50) scaling.scale_on_cpu_utilization( "CpuScaling", target_utilization_percent=85, scale_in_cooldown=core.Duration.seconds(120), scale_out_cooldown=core.Duration.seconds(30), ) # # network acl # database.connections.allow_default_port_from(wordpress_service, "wordpress access to db") file_system.connections.allow_default_port_from(wordpress_service) # # external access # wordpress_service.connections.allow_from( other=properties.load_balancer, port_range=ec2.Port.tcp(80)) http_listener = properties.load_balancer.add_listener( "HttpListener", port=80, ) http_listener.add_targets( "HttpServiceTarget", protocol=elbv2.ApplicationProtocol.HTTP, targets=[wordpress_service], health_check=elbv2.HealthCheck(healthy_http_codes="200,301,302"))
def __init__(self, scope: cdk.Construct, id: str, vpc: ec2.Vpc, s3_bucket_name: str, certificate: acm.Certificate, consoleme_alb: lb.ApplicationLoadBalancer, consoleme_sg: ec2.SecurityGroup, task_role_arn: str, task_execution_role_arn: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # ECS Task definition and volumes if USE_PUBLIC_DOCKER_IMAGE is True: docker_image = ecs.ContainerImage.from_registry(DOCKER_IMAGE) else: docker_image = ecs.ContainerImage.from_docker_image_asset( ecr_assets.DockerImageAsset(self, "ConsoleMeCustomImage", directory="../")) imported_task_role = iam.Role.from_role_arn(self, "ImportedTaskRole", role_arn=task_role_arn) imported_task_execution_role = iam.Role.from_role_arn( self, "ImportedTaskExecutionRole", role_arn=task_execution_role_arn) consoleme_ecs_task_definition = ecs.FargateTaskDefinition( self, "ConsoleMeTaskDefinition", cpu=2048, memory_limit_mib=4096, execution_role=imported_task_execution_role, task_role=imported_task_role, ) # ECS Container definition, service, target group and ALB attachment consoleme_ecs_task_definition.add_container( "Container", image=docker_image, privileged=False, port_mappings=[ ecs.PortMapping(container_port=8081, host_port=8081, protocol=ecs.Protocol.TCP) ], logging=ecs.LogDriver.aws_logs( stream_prefix="ContainerLogs-", log_retention=logs.RetentionDays.ONE_WEEK, ), environment={ "SETUPTOOLS_USE_DISTUTILS": "stdlib", "CONSOLEME_CONFIG_S3": "s3://" + s3_bucket_name + "/config.yaml", "EC2_REGION": self.region, }, working_directory="/apps/consoleme", command=[ "bash", "-c", "python scripts/retrieve_or_decode_configuration.py; python consoleme/__main__.py", ], ) consoleme_ecs_task_definition.add_container( "CeleryContainer", image=docker_image, privileged=False, logging=ecs.LogDriver.aws_logs( stream_prefix="CeleryContainerLogs-", log_retention=logs.RetentionDays.ONE_WEEK, ), environment={ "SETUPTOOLS_USE_DISTUTILS": "stdlib", "CONSOLEME_CONFIG_S3": "s3://" + s3_bucket_name + "/config.yaml", "COLUMNS": "80", "EC2_REGION": self.region, }, command=[ "bash", "-c", "python scripts/retrieve_or_decode_configuration.py; python scripts/initialize_redis_oss.py; celery -A consoleme.celery_tasks.celery_tasks worker -l DEBUG -B -E --concurrency=8", ], ) # ECS cluster cluster = ecs.Cluster(self, "Cluster", vpc=vpc) consoleme_imported_alb = ( lb.ApplicationLoadBalancer. from_application_load_balancer_attributes( self, "ConsoleMeImportedALB", load_balancer_arn=consoleme_alb.load_balancer_arn, vpc=vpc, security_group_id=consoleme_sg.security_group_id, load_balancer_dns_name=consoleme_alb.load_balancer_dns_name, )) consoleme_ecs_service = ecs_patterns.ApplicationLoadBalancedFargateService( self, "Service", cluster=cluster, task_definition=consoleme_ecs_task_definition, load_balancer=consoleme_imported_alb, security_groups=[consoleme_sg], open_listener=False, ) consoleme_ecs_service.target_group.configure_health_check( path="/", enabled=True, healthy_http_codes="200-302") consoleme_ecs_service_scaling_target = applicationautoscaling.ScalableTarget( self, "AutoScalingGroup", max_capacity=MAX_CAPACITY, min_capacity=MIN_CAPACITY, resource_id="service/" + cluster.cluster_name + "/" + consoleme_ecs_service.service.service_name, scalable_dimension="ecs:service:DesiredCount", service_namespace=applicationautoscaling.ServiceNamespace.ECS, role=iam.Role( self, "AutoScaleRole", assumed_by=iam.ServicePrincipal( service="ecs-tasks.amazonaws.com"), description="Role for ECS auto scaling group", managed_policies=[ iam.ManagedPolicy.from_managed_policy_arn( self, "AutoScalingManagedPolicy", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceAutoscaleRole", ) ], ), ) applicationautoscaling.TargetTrackingScalingPolicy( self, "AutoScalingPolicy", scaling_target=consoleme_ecs_service_scaling_target, scale_in_cooldown=cdk.Duration.seconds(amount=10), scale_out_cooldown=cdk.Duration.seconds(amount=10), target_value=50, predefined_metric=applicationautoscaling.PredefinedMetric. ECS_SERVICE_AVERAGE_CPU_UTILIZATION, ) consoleme_imported_alb.add_listener( "ConsoleMeALBListener", protocol=lb.ApplicationProtocol.HTTPS, port=443, certificates=[certificate], default_action=lb.ListenerAction.forward( target_groups=[consoleme_ecs_service.target_group]), )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### S3 ### source_csv_bucket = _s3.Bucket(self, "BYODValidationSourceBucket", versioned=True) target_csv_bucket = _s3.Bucket( self, "BYODValidationTargetBucket", removal_policy=core.RemovalPolicy.RETAIN) webtool_bucket = _s3.Bucket( self, "WebToolBucket", website_index_document="index.html", website_error_document="index.html", public_read_access=True, ) core.CfnOutput(self, "DVTRegion", value=self.region) core.CfnOutput(self, "SourceS3Bucket", value=source_csv_bucket.bucket_name) core.CfnOutput(self, "TargetS3Bucket", value=target_csv_bucket.bucket_name) core.CfnOutput(self, "WebToolS3Bucket", value=webtool_bucket.bucket_name) core.CfnOutput(self, "WebToolUrl", value=webtool_bucket.bucket_website_url) ### Stager Function ### stager_function = _lambda.Function(self, "StagerFunction", runtime=_lambda.Runtime.NODEJS_12_X, code=_lambda.Code.from_asset( os.path.join( dirname, "lambda", "stager")), handler='index.handler') stager_function.add_environment("REGION", self.region) stager_function.add_environment("SOURCE_BUCKET", source_csv_bucket.bucket_name) stager_function.add_environment("STAGE_BUCKET", target_csv_bucket.bucket_name) source_csv_bucket.grant_read(stager_function) target_csv_bucket.grant_put(stager_function) core.CfnOutput(self, "StagerLambdaFunction", value=stager_function.function_name) ### Profiling Queue profiling_job_queue = _sqs.Queue(self, "ProfilingJobQueue") core.CfnOutput(self, "SQSProfileQueue", value=profiling_job_queue.queue_url) ### Cognito ### userpool = _cognito.UserPool(self, "WebToolUserPool", user_pool_name="byod-webtool-userpool", self_sign_up_enabled=True, auto_verify={ "email": True, "phone": False }, user_verification={ "email_subject": "Your verification code", "email_body": "Your verification code is {####}", "email_style": _cognito.VerificationEmailStyle.CODE }, standard_attributes={ "email": { "required": True, "mutable": False } }, password_policy={}) client = userpool.add_client( "webtool-app-client", auth_flows={ "custom": True, "user_password": True, "user_srp": True, #"refresh_token": True }) identity_pool = _cognito.CfnIdentityPool( self, "WebToolCognitoIdentityPool", allow_unauthenticated_identities=True) identity_pool.add_property_override( "CognitoIdentityProviders", [{ "ClientId": client.user_pool_client_id, "ProviderName": userpool.user_pool_provider_name }]) auth_role = _iam.Role( self, "CognitoAuthRole", assumed_by=WebIdentityPrincipal( "cognito-identity.amazonaws.com", { "StringEquals": { "cognito-identity.amazonaws.com:aud": identity_pool.ref }, "ForAnyValue:StringLike": { "cognito-identity.amazonaws.com:amr": "authenticated" } })) auth_role.add_to_policy( PolicyStatement(effect=Effect.ALLOW, actions=["s3:GetObject", "s3:PutObject"], resources=["%s/*" % target_csv_bucket.bucket_arn])) auth_role.add_to_policy( PolicyStatement(effect=Effect.ALLOW, actions=["lambda:invokeFunction"], resources=[stager_function.function_arn])) auth_role.add_to_policy( PolicyStatement(effect=Effect.ALLOW, actions=["sqs:*"], resources=[profiling_job_queue.queue_arn])) unauth_role = _iam.Role( self, "CognitoUnauthRole", assumed_by=_iam.WebIdentityPrincipal( "cognito-identity.amazonaws.com", conditions={ "StringEquals": { "cognito-identity.amazonaws.com:aud": identity_pool.ref }, "ForAnyValue:StringLike": { "cognito-identity.amazonaws.com:amr": "unauthenticated" } })) identity_pool_policy = _cognito.CfnIdentityPoolRoleAttachment( self, "WebToolCognitoIdentityPoolPolicy", identity_pool_id=identity_pool.ref, roles={ 'unauthenticated': unauth_role.role_arn, 'authenticated': auth_role.role_arn }) core.CfnOutput(self, "UserPoolId", value=userpool.user_pool_id) core.CfnOutput(self, "IdentityPoolId", value=identity_pool.ref) core.CfnOutput(self, "ClientId", value=client.user_pool_client_id) core.CfnOutput(self, "ProviderName", value=userpool.user_pool_provider_name) ### DynamoDB ### validation_job_table = _dynamodb.Table( self, "ValidationJobTable", partition_key=_dynamodb.Attribute( name="id", type=_dynamodb.AttributeType.STRING)) ## AppSync ### api = _appsync.GraphqlApi( self, "Api", name="validation-job-api", schema=_appsync.Schema.from_asset( os.path.join(dirname, "api", "schema.graphql")), authorization_config=AuthorizationConfig( default_authorization=AuthorizationMode( authorization_type=AuthorizationType.USER_POOL, user_pool_config=UserPoolConfig(user_pool=userpool))), log_config=LogConfig(exclude_verbose_content=False, field_log_level=FieldLogLevel.ALL)) api_ds = api.add_dynamo_db_data_source("ValidationJobDataSource", validation_job_table) from aws_cdk.aws_appsync import MappingTemplate api_ds.create_resolver( type_name="Query", field_name="listJobss", request_mapping_template=MappingTemplate.from_file( os.path.join(dirname, "api", "resolvers", "Query.listJobss.req.vtl")), response_mapping_template=MappingTemplate.from_file( os.path.join(dirname, "api", "resolvers", "Query.listJobss.res.vtl"))) api_ds.create_resolver( type_name="Query", field_name="getJobs", request_mapping_template=MappingTemplate.from_file( os.path.join(dirname, "api", "resolvers", "Query.getJobs.req.vtl")), response_mapping_template=MappingTemplate.from_file( os.path.join(dirname, "api", "resolvers", "Query.getJobs.res.vtl"))) core.CfnOutput(self, "GraphQLEndpoint", value=api.graphql_url) ### SQS ### validation_job_queue = _sqs.Queue(self, "ValidationJobQueue") ### Lambda ### validation_trigger_function = _lambda.Function( self, "ValidationTriggerFunction", runtime=_lambda.Runtime.PYTHON_3_8, code=_lambda.Code.from_asset( os.path.join(dirname, "lambda", "validation_trigger")), handler='lambda_function.lambda_handler') validation_trigger_function.add_environment( "TABLE_NAME", validation_job_table.table_name) validation_trigger_function.add_environment( "QUEUE_URL", validation_job_queue.queue_url) validation_trigger_function.add_event_source( _S3EventSource(source_csv_bucket, events=[_s3.EventType.OBJECT_CREATED])) source_csv_bucket.grant_read(validation_trigger_function) validation_job_table.grant_read_write_data(validation_trigger_function) validation_job_queue.grant_send_messages(validation_trigger_function) ### ECS Fargate ### validation_fargate_asset = _ecr_assets.DockerImageAsset( self, "ValidationBuildImage", directory=os.path.join(dirname, "fargate", "validation")) profiling_fargate_asset = _ecr_assets.DockerImageAsset( self, "ProfilingBuildImage", directory=os.path.join(dirname, "fargate", "profiling")) vpc = _ec2.Vpc(self, "VPC", max_azs=3) cluster = _ecs.Cluster(self, "ECSCluster", vpc=vpc) validation_fargate_service = _ecs_patterns.QueueProcessingFargateService( self, "ValidationFargateService", cluster=cluster, cpu=4096, memory_limit_mib=30720, enable_logging=True, image=_ecs.ContainerImage.from_docker_image_asset( validation_fargate_asset), environment={ "TABLE_NAME": validation_job_table.table_name, "QUEUE_URL": validation_job_queue.queue_url, "SOURCE_BUCKET_NAME": source_csv_bucket.bucket_name, "TARGET_BUCKET_NAME": target_csv_bucket.bucket_name, "REGION": self.region }, queue=validation_job_queue, max_scaling_capacity=2, max_healthy_percent=200, min_healthy_percent=66) validation_fargate_service.task_definition.task_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonDynamoDBFullAccess")) validation_fargate_service.task_definition.task_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess")) profiling_fargate_service = _ecs_patterns.QueueProcessingFargateService( self, "ProfilingFargateService", cluster=cluster, cpu=4096, memory_limit_mib=30720, enable_logging=True, image=_ecs.ContainerImage.from_docker_image_asset( profiling_fargate_asset), environment={ "TABLE_NAME": validation_job_table.table_name, "QUEUE_URL": profiling_job_queue.queue_url, "SOURCE_BUCKET_NAME": source_csv_bucket.bucket_name, "TARGET_BUCKET_NAME": target_csv_bucket.bucket_name, "REGION": self.region }, queue=profiling_job_queue, max_scaling_capacity=2, max_healthy_percent=200, min_healthy_percent=66) profiling_fargate_service.task_definition.task_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonDynamoDBFullAccess")) profiling_fargate_service.task_definition.task_role.add_managed_policy( _iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3FullAccess"))
def __init__(self, scope: core.Construct, id: str, vpc: ec2.IVpc, *, slaves=2, **kwargs) -> None: super().__init__(scope, id, **kwargs) cluster = ecs.Cluster(self, "cluster", vpc=vpc) locust_asset = ecr_assets.DockerImageAsset(self, 'locust', directory="docker", file="app/Dockerfile") master_task = ecs.FargateTaskDefinition( self, "mastert", cpu=512, memory_limit_mib=1024 ) sg_slave = ec2.SecurityGroup(self, "sgslave", vpc=vpc, allow_all_outbound=True) sg_master = ec2.SecurityGroup(self, "sgmaster", vpc=vpc, allow_all_outbound=True) sg_master.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(8089)) sg_master.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(5557)) master_container = master_task.add_container( "masterc", image=ecs.ContainerImage.from_docker_image_asset(locust_asset), logging=ecs.LogDriver.aws_logs(stream_prefix="master"), command=["-f", "/mnt/locust/locustfile.py", "--master"] ) master_container.add_port_mappings(ecs.PortMapping(container_port=8089, host_port=8089)) master_container.add_port_mappings(ecs.PortMapping(container_port=5557, host_port=5557)) master_service = ecs.FargateService( self, "masters", cluster=cluster, task_definition=master_task, desired_count=1, assign_public_ip=True, security_group=sg_master ) nlb = elbv2.NetworkLoadBalancer( self, "nbalancer", internet_facing=True, vpc=vpc ) listener_master_console = nlb.add_listener( "masterconsole", port=8089, protocol=elbv2.Protocol("TCP") ) listener_console = nlb.add_listener( "master", port=5557, protocol=elbv2.Protocol("TCP") ) listener_master_console.add_targets( "consoletarget", deregistration_delay=core.Duration.seconds(1), port=8089, targets=[master_service.load_balancer_target( container_name="masterc", container_port=8089 )], health_check=elbv2.HealthCheck( healthy_threshold_count=2, unhealthy_threshold_count=2, timeout=core.Duration.seconds(10) ) ) listener_console.add_targets( "mastertarget", deregistration_delay=core.Duration.seconds(1), port=5557, targets=[master_service.load_balancer_target( container_name="masterc", container_port=5557 )], health_check=elbv2.HealthCheck( healthy_threshold_count=2, unhealthy_threshold_count=2, timeout=core.Duration.seconds(10) ) ) slave_task = ecs.FargateTaskDefinition( self, "slavet", cpu=2048, memory_limit_mib=4096 ) slave_task.add_container( "slavec", image=ecs.ContainerImage.from_docker_image_asset(locust_asset), logging=ecs.LogDriver.aws_logs(stream_prefix="slave"), command=["-f", "/mnt/locust/locustfile.py", "--worker", "--master-host", nlb.load_balancer_dns_name] ) ecs.FargateService( self, "slaves", cluster=cluster, task_definition=slave_task, desired_count=slaves, assign_public_ip=True, security_group=sg_slave ) core.CfnOutput(self, "LocustWebConsole", value="http://" + nlb.load_balancer_dns_name + ":8089")
def __init__(self, app: core.Construct, stack_name: str, vpc: aws_ec2.Vpc, security_group: aws_ec2.SecurityGroup): super().__init__(scope=app, id=f"{stack_name}-batch") batch_role = aws_iam.Role( scope=self, id=f"batch_role", role_name=f"batch_role", assumed_by=aws_iam.ServicePrincipal("batch.amazonaws.com")) batch_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AWSBatchServiceRole", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole")) batch_role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["arn:aws:logs:*:*:*"], actions=[ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DescribeLogStreams" ])) # Role to attach EC2 instance_role = aws_iam.Role( scope=self, id=f"instance_role", role_name=f"instance_role_for", assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com")) instance_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AmazonEC2ContainerServiceforEC2Role", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" )) # add policy to access S3 instance_role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["*"], actions=["s3:*"])) # add policy to access CloudWatch Logs instance_role.add_to_policy( aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW, resources=["arn:aws:logs:*:*:*"], actions=[ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:DescribeLogStreams" ])) # attach role to EC2 instance_profile = aws_iam.CfnInstanceProfile( scope=self, id=f"instance_profile", instance_profile_name=f"instance_profile", roles=[instance_role.role_name]) # ===== # # batch # # ===== # batch_compute_resources = aws_batch.ComputeResources( vpc=vpc, maxv_cpus=4, minv_cpus=0, security_groups=[security_group], instance_role=instance_profile.attr_arn, type=aws_batch.ComputeResourceType.SPOT) batch_compute_environment = aws_batch.ComputeEnvironment( scope=self, id="batch_compute_environment", compute_environment_name="batch_compute_environment", compute_resources=batch_compute_resources, service_role=batch_role) job_role = aws_iam.Role( scope=self, id=f"job_role", role_name=f"job_role", assumed_by=aws_iam.ServicePrincipal("ecs-tasks.amazonaws.com")) job_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AmazonECSTaskExecutionRolePolicy", managed_policy_arn= "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" )) job_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"AmazonS3FullAccess", managed_policy_arn="arn:aws:iam::aws:policy/AmazonS3FullAccess" )) job_role.add_managed_policy( aws_iam.ManagedPolicy.from_managed_policy_arn( scope=self, id=f"CloudWatchLogsFullAccess", managed_policy_arn= "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")) self.batch_job_queue = aws_batch.JobQueue( scope=self, id=f"job_queue", job_queue_name=f"job_queue", compute_environments=[ aws_batch.JobQueueComputeEnvironment( compute_environment=batch_compute_environment, order=1) ], priority=1) # ECR repository ecr_repository = aws_ecr_assets.DockerImageAsset( scope=self, id=f"ecr_image", directory="./docker", repository_name=f"repository") # get image from ECR container_image = aws_ecs.ContainerImage.from_ecr_repository( repository=ecr_repository.repository) # job define # pass `S3_BUCKET` as environment argument. self.batch_job_definition = aws_batch.JobDefinition( scope=self, id=f"job_definition", job_definition_name=f"job_definition", container=aws_batch.JobDefinitionContainer( image=container_image, environment={"S3_BUCKET": f"{S3_BUCKET}"}, job_role=job_role, vcpus=1, memory_limit_mib=1024))
def __init__(self, scope: core.Construct, id: str, resources: FsiSharedResources, subnet_group_name: str = 'Default', **kwargs) -> None: super().__init__(scope, id, **kwargs) # Configure the container resources... self.repo = assets.DockerImageAsset(self, 'Repo', directory='src/fsi/earnings', file='Dockerfile') code = lambda_.DockerImageCode.from_ecr( repository=self.repo.repository, tag=self.repo.image_uri.split(':')[-1]) # Configure security policies... role = iam.Role( self, 'Role', assumed_by=iam.ServicePrincipal(service='lambda'), description='HomeNet-{}-Fsi-EarningsReport'.format( resources.landing_zone.zone_name), role_name='fsi-earnings@homenet.{}.{}'.format( resources.landing_zone.zone_name, core.Stack.of(self).region).lower(), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name= 'service-role/AWSLambdaVPCAccessExecutionRole'), ]) # Grant any permissions... self.earnings_table = d.Table( self, 'EarningCalendar', table_name='FsiCoreSvc-EarningsCalendar', billing_mode=d.BillingMode.PAY_PER_REQUEST, partition_key=d.Attribute(name='PartitionKey', type=d.AttributeType.STRING), sort_key=d.Attribute(name='SortKey', type=d.AttributeType.STRING), time_to_live_attribute='Expiration', point_in_time_recovery=True, server_side_encryption=True) self.earnings_table.grant_read_write_data(role) # Define any variables for the function self.function_env = { 'CACHE_TABLE': self.earnings_table.table_name, } # Create the backing webapi compute ... self.function = lambda_.DockerImageFunction( self, 'Function', code=code, role=role, function_name='HomeNet-{}-Fsi-{}'.format( resources.landing_zone.zone_name, FsiEarningsGateway.__name__), description='Python Lambda function for ' + FsiEarningsGateway.__name__, timeout=core.Duration.seconds(30), tracing=lambda_.Tracing.ACTIVE, vpc=resources.landing_zone.vpc, log_retention=logs.RetentionDays.FIVE_DAYS, memory_size=128, allow_all_outbound=True, vpc_subnets=ec2.SubnetSelection( subnet_group_name=subnet_group_name), security_groups=[resources.landing_zone.security_group], environment=self.function_env, ) # Bind APIG to Lambda compute... self.frontend_proxy = a.LambdaRestApi( self, 'ApiGateway', proxy=True, handler=self.function, options=a.RestApiProps( description='Hosts the Earnings Calendar Services via ' + self.function.function_name, domain_name=a.DomainNameOptions( domain_name='earnings.trader.fsi', certificate=Certificate.from_certificate_arn( self, 'Certificate', certificate_arn= 'arn:aws:acm:us-east-2:581361757134:certificate/4e3235f7-49a1-42a5-a671-f2449b45f72d' ), security_policy=a.SecurityPolicy.TLS_1_0), policy=iam.PolicyDocument(statements=[ iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=['execute-api:Invoke'], principals=[iam.AnyPrincipal()], resources=['*'], conditions={ 'IpAddress': { 'aws:SourceIp': [ '10.0.0.0/8', '192.168.0.0/16', '72.90.160.65/32' ] } }) ]), endpoint_configuration=a.EndpointConfiguration( types=[a.EndpointType.REGIONAL], ))) # Register Dns Name r53.ARecord(self, 'AliasRecord', zone=resources.trader_dns_zone, record_name='earnings.%s' % resources.trader_dns_zone.zone_name, target=r53.RecordTarget.from_alias( dns_targets.ApiGateway(self.frontend_proxy)))
def __init__(self, scope: core.Construct, id: str, datalake: DataLakeLayer, **kwargs) -> None: super().__init__(scope, id, **kwargs) self.__datalake = datalake self.security_group = ec2.SecurityGroup( self, 'SecurityGroup', vpc=self.datalake.vpc, allow_all_outbound=True, description='SonarQube Security Group') self.security_group.add_ingress_rule(peer=ec2.Peer.any_ipv4(), connection=ec2.Port.all_traffic(), description='Allow any traffic') self.sonarqube_svr_ecr = ecr.DockerImageAsset( self, 'Repo', directory=os.path.join(root_dir, 'images/sonarqube-server'), repository_name='sonarqube') self.sonarqube_cli_ecr = ecr.DockerImageAsset( self, 'Cli', directory=os.path.join(root_dir, 'images/sonarqube-scanner'), repository_name='sonarqube-cli') self.database = rds.DatabaseCluster( self, 'Database', engine=rds.DatabaseClusterEngine.aurora_postgres( version=rds.AuroraPostgresEngineVersion.VER_11_9), default_database_name='sonarqube', removal_policy=core.RemovalPolicy.DESTROY, credentials=rds.Credentials.from_username( username='******', password=core.SecretValue(value='postgres')), instance_props=rds.InstanceProps( vpc=self.datalake.vpc, security_groups=[self.security_group], instance_type=ec2.InstanceType('r6g.xlarge'))) # self.ecs_cluster = ecs.Cluster(self,'SonarCluster', # container_insights=True, # vpc=self.datalake.vpc, # capacity=ecs.AddCapacityOptions( # machine_image_type= ecs.MachineImageType.AMAZON_LINUX_2, # instance_type=ec2.InstanceType('m5.xlarge'), # allow_all_outbound=True, # associate_public_ip_address=False, # vpc_subnets= ec2.SubnetSelection(subnet_type= ec2.SubnetType.PUBLIC), # desired_capacity=2)) # self.service = ecsp.ApplicationLoadBalancedEc2Service(self,'SonarEc2', # cluster=self.ecs_cluster, # desired_count=1, # listener_port=80, # memory_reservation_mib= 4 * 1024, # task_image_options= ecsp.ApplicationLoadBalancedTaskImageOptions( # image= ecs.ContainerImage.from_docker_image_asset(asset=self.sonarqube_svr_ecr), # container_name='sonarqube-svr', # container_port=9000, # enable_logging=True, # environment={ # '_SONAR_JDBC_URL':'jdbc:postgresql://{}/sonarqube'.format( # self.database.cluster_endpoint.hostname), # '_SONAR_JDBC_USERNAME':'******', # '_SONAR_JDBC_PASSWORD':'******' # })) self.service = ecsp.ApplicationLoadBalancedFargateService( self, 'Server', assign_public_ip=True, vpc=self.datalake.vpc, desired_count=1, cpu=4096, memory_limit_mib=8 * 1024, listener_port=80, platform_version=ecs.FargatePlatformVersion.VERSION1_4, security_groups=[self.security_group, self.datalake.efs_sg], task_image_options=ecsp.ApplicationLoadBalancedTaskImageOptions( image=ecs.ContainerImage.from_docker_image_asset( asset=self.sonarqube_svr_ecr), container_name='sonarqube-svr', container_port=9000, enable_logging=True, environment={ '_SONAR_JDBC_URL': 'jdbc:postgresql://{}/sonarqube'.format( self.database.cluster_endpoint.hostname), '_SONAR_JDBC_USERNAME': '******', '_SONAR_JDBC_PASSWORD': '******' })) for name in ['AmazonElasticFileSystemClientFullAccess']: self.service.task_definition.task_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name(name)) # Override container specific settings container = self.service.task_definition.default_container # Required to start remote sql container.add_ulimits( ecs.Ulimit(name=ecs.UlimitName.NOFILE, soft_limit=262145, hard_limit=262145)) for folder in ['data', 'logs']: efs_ap = self.datalake.efs.add_access_point( 'sonarqube-' + folder, create_acl=efs.Acl(owner_gid="0", owner_uid="0", permissions="777"), path='/sonarqube/' + folder) self.service.task_definition.add_volume( name=folder, efs_volume_configuration=ecs.EfsVolumeConfiguration( file_system_id=self.datalake.efs.file_system_id, transit_encryption='ENABLED', authorization_config=ecs.AuthorizationConfig( access_point_id=efs_ap.access_point_id, iam='DISABLED'))) container.add_mount_points( ecs.MountPoint(container_path='/opt/sonarqube/' + folder, source_volume=folder, read_only=False))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # create ecr repo repository = aws_ecr.Repository( self, "dbfit", image_scan_on_push=True, repository_name="dbfit", ) # add image to repo dockerImageAsset = aws_ecr_assets.DockerImageAsset( self, "dbfitImg", directory="./", repository_name="dbfit", exclude=['node_modules', '.git', 'cdk.out']) # create task definition task_definition = aws_ecs.TaskDefinition( self, "dbfittask", memory_mib="512", cpu="256", network_mode=aws_ecs.NetworkMode.AWS_VPC, compatibility=aws_ecs.Compatibility.EC2_AND_FARGATE) # get image from ecr repo container_image = aws_ecs.ContainerImage.from_docker_image_asset( dockerImageAsset) # add container to task definition container = task_definition.add_container("dbfit-container", image=container_image, memory_reservation_mib=512, cpu=256) # add port mapping to expose to outside world port_mapping = aws_ecs.PortMapping(container_port=8085) container.add_port_mappings(port_mapping) # get vpc reference vpc = aws_ec2.Vpc.from_lookup( self, "VPC", # This imports the default VPC but you can also # specify a 'vpcName' or 'tags'. is_default=True) # create cluster # Create an ECS cluster cluster = aws_ecs.Cluster(self, "Cluster", vpc=vpc) # create security group for ecs service security_group = aws_ec2.SecurityGroup( self, "dbfitSG", vpc=vpc, security_group_name="dbfitGroup", allow_all_outbound=True) # open 8085 inbound security_group.add_ingress_rule(aws_ec2.Peer.ipv4('0.0.0.0/0'), aws_ec2.Port.tcp(8085), 'DBFit Routing') # start fargate service on cluster fargate_service = aws_ecs.FargateService( self, "dbfitservice", cluster=cluster, task_definition=task_definition, assign_public_ip=True, security_groups=[security_group])
def __init__(self, scope: core.Construct, id: builtins.str, resources:FsiSharedResources, subnet_group_name:str='Default') -> None: super().__init__(scope, id) self.__resources = resources self.datastores = FsiCollectionDataStoreConstruct(self,'DataStores', resources=self.resources, subnet_group_name=subnet_group_name) self.eventing = FsiCollectionsEventing(self,'Eventing', landing_zone=resources.landing_zone) # Configure role... role = iam.Role(self,'Role', assumed_by=iam.ServicePrincipal(service='lambda'), description='Ameritrade Data Collection Lambda via '+self.component_name, role_name='{}@homenet-{}.{}'.format( self.component_name, resources.landing_zone.zone_name, core.Stack.of(self).region), managed_policies=[ iam.ManagedPolicy.from_aws_managed_policy_name( managed_policy_name='service-role/AWSLambdaVPCAccessExecutionRole') ]) resources.tda_secret.grant_read(role) self.datastores.instrument_table.grant_read_write_data(role) self.datastores.transaction_table.grant_read_write_data(role) self.datastores.quotes_table.grant_read_write_data(role) self.datastores.options_table.grant_read_write_data(role) # Configure the lambda... self.repo = assets.DockerImageAsset(self,'Repo', directory=source_directory) code = lambda_.DockerImageCode.from_ecr( repository=self.repo.repository, tag=self.repo.image_uri.split(':')[-1]) self.function = lambda_.DockerImageFunction(self,'Function', code = code, role= role, function_name='HomeNet-Fsi{}-{}'.format( resources.landing_zone.zone_name, self.component_name), description='Python container function for '+self.component_name, timeout= core.Duration.minutes(15), tracing= lambda_.Tracing.ACTIVE, vpc= resources.landing_zone.vpc, log_retention= RetentionDays.TWO_WEEKS, memory_size= 3 * 128, allow_all_outbound=True, vpc_subnets=ec2.SubnetSelection(subnet_group_name=subnet_group_name), security_groups=[resources.landing_zone.security_group], environment={ 'REGION':core.Stack.of(self).region, 'TDA_SECRET_ID': resources.tda_secret.secret_arn, 'TDA_REDIRECT_URI': ssm.StringParameter.from_string_parameter_name(self,'TDA_REDIRECT_URI', string_parameter_name='/HomeNet/Amertitrade/redirect_uri').string_value, 'TDA_CLIENT_ID': ssm.StringParameter.from_string_parameter_name(self, 'TDA_CLIENT_ID', string_parameter_name='/HomeNet/Ameritrade/client_id').string_value, 'INSTRUMENT_TABLE_NAME': self.datastores.instrument_table.table_name, 'TRANSACTION_TABLE_NAME': self.datastores.transaction_table.table_name, 'QUOTES_TABLE_NAME': self.datastores.quotes_table.table_name, 'OPTIONS_TABLE_NAME': self.datastores.options_table.table_name, } ) # Define the execution schedule... self.add_states_schedule('DiscoverInstruments', schedule=events.Schedule.cron(week_day='SUN',hour="0", minute="0")) self.add_states_schedule('DiscoverOptionable', schedule=events.Schedule.cron(week_day='SUN',hour="1", minute="0")) self.add_states_schedule('CollectFundamentals', schedule=events.Schedule.cron(week_day='SUN',hour="2", minute="0")) self.add_states_schedule('CollectIntraday', schedule=events.Schedule.cron(week_day='MON-FRI',hour="13-23/3", minute="0"), payload={ 'Action': 'CollectHistoric', 'CandleConfiguration':{ 'period_type':'day', 'period':'1', 'frequency_type':'minute', 'frequency':'1' } }) self.add_states_schedule('CollectOptions', schedule=events.Schedule.cron(week_day='MON-FRI',hour="12-23/3", minute="0")) self.add_states_schedule('CollectClosingBell', schedule=events.Schedule.cron(week_day='MON-FRI',hour="23", minute="0"), payload={ 'Action': 'CollectHistoric', 'CandleConfiguration':{ 'period_type':'day', 'period':'1', 'frequency_type':'minute', 'frequency':'1' } }) self.add_states_schedule('CollectHistoric', schedule=events.Schedule.cron(week_day='SAT',hour="0", minute="0"), payload={ 'Action': 'CollectHistoric', 'CandleConfiguration':{ 'period_type':'year', 'period':'20', 'frequency_type':'daily', 'frequency':'1' } }) self.add_states_schedule('CollectTransactions', schedule=events.Schedule.cron(week_day='SUN-FRI', minute="30"))
def __init__( self, scope: core.Construct, id: str, vpc_stack, logstash_ec2=True, logstash_fargate=True, **kwargs, ) -> None: super().__init__(scope, id, **kwargs) # get s3 bucket name s3client = boto3.client("s3") s3_bucket_list = s3client.list_buckets() s3_bucket_name = "" for bkt in s3_bucket_list["Buckets"]: try: bkt_tags = s3client.get_bucket_tagging( Bucket=bkt["Name"])["TagSet"] for keypairs in bkt_tags: if (keypairs["Key"] == "aws:cloudformation:stack-name" and keypairs["Value"] == "elkk-athena"): s3_bucket_name = bkt["Name"] except ClientError as err: if err.response["Error"]["Code"] in [ "NoSuchTagSet", "NoSuchBucket" ]: pass else: print(f"Unexpected error: {err}") # get elastic endpoint esclient = boto3.client("es") es_domains = esclient.list_domain_names() try: es_domain = [ dom["DomainName"] for dom in es_domains["DomainNames"] if "elkk-" in dom["DomainName"] ][0] es_endpoint = esclient.describe_elasticsearch_domain( DomainName=es_domain) es_endpoint = es_endpoint["DomainStatus"]["Endpoints"]["vpc"] except IndexError: es_endpoint = "" # assets for logstash stack logstash_yml = assets.Asset(self, "logstash_yml", path=os.path.join(dirname, "logstash.yml")) logstash_repo = assets.Asset(self, "logstash_repo", path=os.path.join(dirname, "logstash.repo")) # update conf file to .asset # kafka brokerstring does not need reformatting logstash_conf_asset = file_updated( os.path.join(dirname, "logstash.conf"), { "$s3_bucket": s3_bucket_name, "$es_endpoint": es_endpoint, "$kafka_brokers": kafka_get_brokers(), "$elkk_region": os.environ["CDK_DEFAULT_REGION"], }, ) logstash_conf = assets.Asset( self, "logstash.conf", path=logstash_conf_asset, ) # logstash security group logstash_security_group = ec2.SecurityGroup( self, "logstash_security_group", vpc=vpc_stack.get_vpc, description="logstash security group", allow_all_outbound=True, ) core.Tags.of(logstash_security_group).add("project", constants["PROJECT_TAG"]) core.Tags.of(logstash_security_group).add("Name", "logstash_sg") # Open port 22 for SSH logstash_security_group.add_ingress_rule( ec2.Peer.ipv4(f"{external_ip}/32"), ec2.Port.tcp(22), "from own public ip", ) # get security group for kafka ec2client = boto3.client("ec2") security_groups = ec2client.describe_security_groups(Filters=[{ "Name": "tag-value", "Values": [constants["PROJECT_TAG"]] }], ) # if kafka sg does not exist ... don't add it try: kafka_sg_id = [ sg["GroupId"] for sg in security_groups["SecurityGroups"] if "kafka security group" in sg["Description"] ][0] kafka_security_group = ec2.SecurityGroup.from_security_group_id( self, "kafka_security_group", security_group_id=kafka_sg_id) # let in logstash kafka_security_group.connections.allow_from( logstash_security_group, ec2.Port.all_traffic(), "from logstash", ) except IndexError: # print("kafka_sg_id and kafka_security_group not found") pass # get security group for elastic try: elastic_sg_id = [ sg["GroupId"] for sg in security_groups["SecurityGroups"] if "elastic security group" in sg["Description"] ][0] elastic_security_group = ec2.SecurityGroup.from_security_group_id( self, "elastic_security_group", security_group_id=elastic_sg_id) # let in logstash elastic_security_group.connections.allow_from( logstash_security_group, ec2.Port.all_traffic(), "from logstash", ) except IndexError: pass # elastic policy access_elastic_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "es:ListDomainNames", "es:DescribeElasticsearchDomain", "es:ESHttpPut", ], resources=["*"], ) # kafka policy access_kafka_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["kafka:ListClusters", "kafka:GetBootstrapBrokers"], resources=["*"], ) # s3 policy access_s3_policy = iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["s3:ListBucket", "s3:PutObject"], resources=["*"], ) # create the Logstash instance if logstash_ec2: # userdata for Logstash logstash_userdata = user_data_init( log_group_name="elkk/logstash/instance") # create the instance logstash_instance = ec2.Instance( self, "logstash_client", instance_type=ec2.InstanceType(constants["LOGSTASH_INSTANCE"]), machine_image=ec2.AmazonLinuxImage( generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2), vpc=vpc_stack.get_vpc, vpc_subnets=SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), key_name=constants["KEY_PAIR"], security_group=logstash_security_group, user_data=logstash_userdata, ) core.Tag.add(logstash_instance, "project", constants["PROJECT_TAG"]) # add access to the file assets logstash_yml.grant_read(logstash_instance) logstash_repo.grant_read(logstash_instance) logstash_conf.grant_read(logstash_instance) # add permissions to instance logstash_instance.add_to_role_policy( statement=access_elastic_policy) logstash_instance.add_to_role_policy(statement=access_kafka_policy) logstash_instance.add_to_role_policy(statement=access_s3_policy) # add log permissions instance_add_log_permissions(logstash_instance) # add commands to the userdata logstash_userdata.add_commands( # get setup assets files f"aws s3 cp s3://{logstash_yml.s3_bucket_name}/{logstash_yml.s3_object_key} /home/ec2-user/logstash.yml", f"aws s3 cp s3://{logstash_repo.s3_bucket_name}/{logstash_repo.s3_object_key} /home/ec2-user/logstash.repo", f"aws s3 cp s3://{logstash_conf.s3_bucket_name}/{logstash_conf.s3_object_key} /home/ec2-user/logstash.conf", # install java "amazon-linux-extras install java-openjdk11 -y", # install git "yum install git -y", # install pip "yum install python-pip -y", # get elastic output to es "git clone https://github.com/awslabs/logstash-output-amazon_es.git /home/ec2-user/logstash-output-amazon_es", # logstash "rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch", # move logstash repo file "mv -f /home/ec2-user/logstash.repo /etc/yum.repos.d/logstash.repo", # get to the yum "yum install logstash -y", # add user to logstash group "usermod -a -G logstash ec2-user", # move logstash.yml to final location "mv -f /home/ec2-user/logstash.yml /etc/logstash/logstash.yml", # move logstash.conf to final location "mv -f /home/ec2-user/logstash.conf /etc/logstash/conf.d/logstash.conf", # move plugin "mkdir /usr/share/logstash/plugins", "mv -f /home/ec2-user/logstash-output-amazon_es /usr/share/logstash/plugins/logstash-output-amazon_es", # update gemfile """sed -i '5igem "logstash-output-amazon_es", :path => "/usr/share/logstash/plugins/logstash-output-amazon_es"' /usr/share/logstash/Gemfile""", # update ownership "chown -R logstash:logstash /etc/logstash", # start logstash "systemctl start logstash.service", ) # add the signal logstash_userdata.add_signal_on_exit_command( resource=logstash_instance) # add creation policy for instance logstash_instance.instance.cfn_options.creation_policy = core.CfnCreationPolicy( resource_signal=core.CfnResourceSignal(count=1, timeout="PT10M")) # fargate for logstash if logstash_fargate: # cloudwatch log group for containers logstash_logs_containers = logs.LogGroup( self, "logstash_logs_containers", log_group_name="elkk/logstash/container", removal_policy=core.RemovalPolicy.DESTROY, retention=logs.RetentionDays.ONE_WEEK, ) # docker image for logstash logstash_image_asset = ecr_assets.DockerImageAsset( self, "logstash_image_asset", directory=dirname # , file="Dockerfile" ) # create the fargate cluster logstash_cluster = ecs.Cluster(self, "logstash_cluster", vpc=vpc_stack.get_vpc) core.Tag.add(logstash_cluster, "project", constants["PROJECT_TAG"]) # the task logstash_task = ecs.FargateTaskDefinition( self, "logstash_task", cpu=512, memory_limit_mib=1024, ) # add container to the task logstash_task.add_container( logstash_image_asset.source_hash, image=ecs.ContainerImage.from_docker_image_asset( logstash_image_asset), logging=ecs.LogDrivers.aws_logs( stream_prefix="elkk", log_group=logstash_logs_containers), ) # add permissions to the task logstash_task.add_to_task_role_policy(access_s3_policy) logstash_task.add_to_task_role_policy(access_elastic_policy) # the service logstash_service = (ecs.FargateService( self, "logstash_service", cluster=logstash_cluster, task_definition=logstash_task, security_group=logstash_security_group, deployment_controller=ecs.DeploymentController( type=ecs.DeploymentControllerType.ECS), ).auto_scale_task_count( min_capacity=3, max_capacity=10).scale_on_cpu_utilization( "logstash_scaling", target_utilization_percent=75, scale_in_cooldown=core.Duration.seconds(60), scale_out_cooldown=core.Duration.seconds(60), ))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here #vpc = ec2.Vpc.from_lookup(self, 'VPC', is_default=True) vpc = ec2.Vpc(self, "MyVpc", max_azs=2) rdsInst = rds.DatabaseInstance( self, 'SpringPetclinicDB', engine=rds.DatabaseInstanceEngine.MYSQL, instance_class=ec2.InstanceType('t2.medium'), master_username='******', database_name='petclinic', master_user_password=core.SecretValue('Welcome#123456'), vpc=vpc, deletion_protection=False, backup_retention=core.Duration.days(0), removal_policy=core.RemovalPolicy.DESTROY, #vpc_placement = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC) ) rdsInst.connections.allow_default_port_from_any_ipv4() cluster = ecs.Cluster(self, 'EcsCluster', vpc=vpc) asset = ecr_assets.DockerImageAsset( self, 'spring-petclinic', directory='./docker/', build_args={ 'JAR_FILE': 'spring-petclinic-2.1.0.BUILD-SNAPSHOT.jar' }) cluster.add_capacity( "DefaultAutoScalingGroup", instance_type=ec2.InstanceType('t2.large'), vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), min_capacity=2) ecs_service = ecs_patterns.ApplicationLoadBalancedEc2Service( self, "Ec2Service", cluster=cluster, memory_limit_mib=1024, service_name='spring-petclinic', desired_count=2, task_image_options={ "image": ecs.ContainerImage.from_docker_image_asset(asset), "container_name": 'spring-petclinic', "container_port": 8080, "environment": { 'SPRING_DATASOURCE_PASSWORD': '******', 'SPRING_DATASOURCE_USERNAME': '******', 'SPRING_PROFILES_ACTIVE': 'mysql', 'SPRING_DATASOURCE_URL': 'jdbc:mysql://' + rdsInst.db_instance_endpoint_address + '/petclinic?useUnicode=true' } })