def __init__(self, scope: Construct, id: str, *, deployment: Deployment, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) func = lambda_edge.create_function( self, f"Preview{deployment.value}IndexRedirect", runtime=Runtime.NODEJS_10_X, handler="index.handler", code=Code.from_asset("./lambdas/preview-redirect"), ) s3_cloud_front = S3CloudFront( self, "S3CloudFront", subdomain_name=self.subdomain_name, error_folder="/errors", lambda_function_associations=[ LambdaFunctionAssociation( event_type=LambdaEdgeEventType.ORIGIN_REQUEST, lambda_function=func, ), ], ) S3CloudFrontPolicy( self, "S3cloudFrontPolicy", s3_cloud_front=s3_cloud_front, )
def get_cluster(self, vpc): cluster = Cluster(self._stack, self._name, vpc=vpc) Tags.of(cluster).add('hostedZoneId', self._config.hosted_zone_id) Tags.of(cluster).add('domain', self._config.domain) self._export('ClusterName', cluster.cluster_name) self._tag_it(cluster) return cluster
def add_tags_to_stack(stack: Stack, config: Dict) -> None: # Add common tags for tag_key in config['tags']: Tags.of(stack).add(key=tag_key, value=config['tags'][tag_key]) # Add environment in the tags Tags.of(stack).add(key='stage', value=config['stage'])
def __init__(self, scope: Construct, id: str, context: "Context", team_context: "TeamContext", parameters: Dict[str, Any]) -> None: super().__init__( scope=scope, id=id, stack_name=id, env=Environment(account=context.account_id, region=context.region), ) Tags.of(scope=cast(IConstruct, self)).add( key="Env", value=f"orbit-{context.name}") if team_context.eks_pod_role_arn is None: raise ValueError("Pod Role arn required") team_role = iam.Role.from_role_arn( scope=self, id="team-role", role_arn=team_context.eks_pod_role_arn, mutable=True) team_role.attach_inline_policy(policy=iam.Policy( scope=self, id="emr_on_eks", policy_name="emr_on_eks", statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "emr-containers:StartJobRun", "emr-containers:ListJobRuns", "emr-containers:DescribeJobRun", "emr-containers:CancelJobRun", "emr-containers:TagResource", ], resources=[parameters.get("virtual_arn", "*")], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "logs:*", ], resources=[ f"arn:aws:logs:{context.region}:{context.account_id}:log-group:/orbit/emr/*", f"arn:aws:logs:{context.region}:{context.account_id}:log-group:/orbit/emr/*:log-stream:*", ], ), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=[ "emr-containers:Get*", "emr-containers:Describe*", "emr-containers:List*", "elasticmapreduce:CreatePersistentAppUI", "elasticmapreduce:DescribePersistentAppUI", "elasticmapreduce:GetPersistentAppUIPresignedURL", ], resources=["*"], ), ], ))
def __init__(self, scope: Construct, stack_id: str, deploy_env: str, **kwargs: Any) -> None: super().__init__(scope, stack_id, **kwargs) ############################################################################################ # ### NETWORKING ########################################################################### ############################################################################################ # create new VPC aws_ec2.Vpc( self, "datalake", # cidr='10.0.0.0/16', # TODO: use specific CIDR subnet_configuration=[ aws_ec2.SubnetConfiguration( cidr_mask=27, name="public", subnet_type=aws_ec2.SubnetType.PUBLIC), aws_ec2.SubnetConfiguration( cidr_mask=20, name="ecs-cluster", subnet_type=aws_ec2.SubnetType.PRIVATE), aws_ec2.SubnetConfiguration( name="reserved", subnet_type=aws_ec2.SubnetType.PRIVATE, reserved=True, ), ], max_azs=99 if deploy_env == "prod" else 1, ) Tags.of(self).add("ApplicationLayer", "networking") # type: ignore[arg-type]
def __init__( self, scope: Construct, id: str, context: "Context", team_context: "TeamContext", parameters: Dict[str, Any], ) -> None: super().__init__( scope=scope, id=id, stack_name=id, env=Environment(account=context.account_id, region=context.region), ) Tags.of(scope=cast(IConstruct, self)).add( key="Env", value=f"orbit-{context.name}") _logger.info(f"Plugin parameters: {parameters}") # just showing how to create resource. Do not forget to update the IAM policy or make sure the attached policy # for the team is allowing the creation and destruction of the resource. ssm_parameter: str = f"/orbit/{context.name}/{team_context.name}/hello-plugin" ssm.StringParameter( scope=self, id="param", string_value="testing plugin hello world", parameter_name=ssm_parameter, )
def __init__(self, scope: Construct, stack_id: str, env_name: str) -> None: super().__init__(scope, stack_id) ############################################################################################ # ### NETWORKING ########################################################################### ############################################################################################ # create new VPC aws_ec2.Vpc( self, "geostore", # cidr='10.0.0.0/16', # TODO: use specific CIDR subnet_configuration=[ aws_ec2.SubnetConfiguration( cidr_mask=27, name="public", subnet_type=aws_ec2.SubnetType.PUBLIC ), aws_ec2.SubnetConfiguration( cidr_mask=20, name="ecs-cluster", subnet_type=aws_ec2.SubnetType.PRIVATE ), aws_ec2.SubnetConfiguration( name="reserved", subnet_type=aws_ec2.SubnetType.PRIVATE, reserved=True, ), ], max_azs=99 if env_name == PRODUCTION_ENVIRONMENT_NAME else 1, ) Tags.of(self).add("ApplicationLayer", "networking") # type: ignore[arg-type]
def __init__(self, scope: Construct, id: str, *, deployment: Deployment, policy: Policy, cluster: ICluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) policy.add_stack(self) if deployment == Deployment.PRODUCTION: desired_count = 2 priority = 50 else: desired_count = 1 priority = 150 ECSHTTPSContainer( self, self.application_name, subdomain_name=self.subdomain_name, deployment=deployment, policy=policy, application_name=self.application_name, image_name="ghcr.io/openttd/binaries-redirect", port=80, memory_limit_mib=16, desired_count=desired_count, cluster=cluster, priority=priority, allow_via_http=True, )
def __init__( self, scope: Construct, stack_id: str, *, deploy_env: str, storage_bucket: aws_s3.Bucket, **kwargs: Any, ) -> None: super().__init__(scope, stack_id, **kwargs) account_principal = aws_iam.AccountPrincipal(account_id=276514628126) role = aws_iam.Role( self, "koordinates-read-role", role_name=f"koordinates-s3-access-read-{deploy_env}", assumed_by=account_principal, # type: ignore[arg-type] external_id={"prod": "koordinates-jAddR"}.get(deploy_env, "koordinates-4BnJQ"), max_session_duration=MAX_SESSION_DURATION, ) storage_bucket.grant_read(role) # type: ignore[arg-type] Tags.of(self).add("ApplicationLayer", "lds") # type: ignore[arg-type]
def __init__(self, scope: Construct, stack_id: str, *, deploy_env: str, **kwargs: Any) -> None: super().__init__(scope, stack_id, **kwargs) ############################################################################################ # ### DATASET STAGING S3 BUCKET ############################################################ ############################################################################################ self.staging_bucket = aws_s3.Bucket( self, "dataset-staging-bucket", access_control=aws_s3.BucketAccessControl.PRIVATE, block_public_access=aws_s3.BlockPublicAccess.BLOCK_ALL, versioned=True, removal_policy=RemovalPolicy.DESTROY, ) self.staging_bucket_name_parameter = aws_ssm.StringParameter( self, "staging bucket name", description=f"Staging bucket name for {deploy_env}", parameter_name=ParameterName.STAGING_BUCKET_NAME.value, string_value=self.staging_bucket.bucket_name, ) Tags.of(self).add("ApplicationLayer", "staging") # type: ignore[arg-type]
def __init__(self, scope: Construct, id: str, *, deployment: Deployment, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) bucket_site = Bucket( self, "Site", block_public_access=BlockPublicAccess.BLOCK_ALL, ) bucket_access_logs = Bucket( self, "AccessLogs", encryption=BucketEncryption.S3_MANAGED, block_public_access=BlockPublicAccess.BLOCK_ALL, ) for subdomain_name in self.subdomain_names: func_version = lambda_edge.create_function( self, f"Redirect-{subdomain_name}-{deployment.value}", runtime=Runtime.NODEJS_10_X, handler="index.handler", code=Code.from_asset(f"./lambdas/redirect-{subdomain_name}"), ) if subdomain_name == "grfsearch": S3CloudFrontV2( self, f"S3CloudFront-{subdomain_name}", subdomain_name=subdomain_name, bucket_site=bucket_site, bucket_access_logs=bucket_access_logs, edge_lambdas=[ EdgeLambda( event_type=LambdaEdgeEventType.ORIGIN_REQUEST, function_version=func_version, ), ], forward_query_string=True, forward_query_string_cache_keys=["do", "q"], ) else: S3CloudFront( self, f"S3CloudFront-{subdomain_name}", subdomain_name=subdomain_name, bucket_site=bucket_site, bucket_access_logs=bucket_access_logs, lambda_function_associations=[ LambdaFunctionAssociation( event_type=LambdaEdgeEventType.ORIGIN_REQUEST, lambda_function=func_version, ), ], )
def _create_cluster_pod_security_group(self) -> ec2.SecurityGroup: name = f"orbit-{self.context.name}-cluster-pod-sg" sg = ec2.SecurityGroup( scope=self, id="cluster-pod-security-group", security_group_name=name, vpc=self.i_vpc, ) Tags.of(scope=sg).add(key="Name", value=name) return sg
def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) global g_tasks Tags.of(self).add("Stack", "Common-Tasks") if g_tasks is not None: raise Exception("Only a single TasksStack instance can exist") g_tasks = self
def __init__(self, scope: Construct, id: str, *, deployment: Deployment, policy: Policy, cluster: ICluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) policy.add_stack(self) if deployment == Deployment.PRODUCTION: desired_count = 1 # Currently this pod is stateful, and as such cannot be run more than once priority = 46 else: desired_count = 1 priority = 146 api_fqdn = dns.subdomain_to_fqdn("api.bananas") api_url = f"https://{api_fqdn}" frontend_fqdn = dns.subdomain_to_fqdn(self.subdomain_name) frontend_url = f"https://{frontend_fqdn}" sentry_dsn = parameter_store.add_secure_string( f"/BananasFrontendWeb/{deployment.value}/SentryDSN").parameter ECSHTTPSContainer( self, self.application_name, subdomain_name=self.subdomain_name, deployment=deployment, policy=policy, application_name=self.application_name, image_name="ghcr.io/openttd/bananas-frontend-web", port=80, memory_limit_mib=64, desired_count=desired_count, cluster=cluster, priority=priority, command=[ "--api-url", api_url, "--frontend-url", frontend_url, "run", "-p", "80", "-h", "0.0.0.0", ], environment={ "WEBCLIENT_SENTRY_ENVIRONMENT": deployment.value.lower(), }, secrets={ "WEBCLIENT_SENTRY_DSN": Secret.from_ssm_parameter(sentry_dsn), }, )
def __init__(self, scope: Construct, id: str, *, alb: IApplicationLoadBalancer, **kwargs) -> None: super().__init__(scope, id, **kwargs) global g_listener_https Tags.of(self).add("Stack", "Common-Listener-Https") self._used_priorities = [] self._subdomains_cert = {} self._alb = alb self._listener = ApplicationListener( self, "Listener-Https", load_balancer=alb, port=443, protocol=ApplicationProtocol.HTTPS, ) # By default, only IPv4 is added to allowed connections self._listener.connections.allow_default_port_from( other=Peer.any_ipv6(), description="Allow from anyone on port 443", ) # Make sure there is always a backend picking up, even if we don't know the host self._listener.add_fixed_response( "default", status_code="404", message_body="Page not found", ) # Add a redirect; in case people go to HTTP, redirect them to HTTPS. self._http_listener = ApplicationListener( self, "Listener-Http", load_balancer=alb, port=80, protocol=ApplicationProtocol.HTTP, ) self._http_listener.connections.allow_default_port_from( other=Peer.any_ipv6(), description="Allow from anyone on port 80", ) self._http_listener.add_redirect_response( "Http-To-Https", status_code="HTTP_301", port="443", protocol="HTTPS", ) if g_listener_https is not None: raise Exception( "Only a single ListenerHTTPSStack instance can exist") g_listener_https = self
def __init__( self, scope: Construct, id: str, context: "Context", ) -> None: self.scope = scope self.id = id self.context = context super().__init__( scope=scope, id=id, stack_name=id, env=Environment(account=self.context.account_id, region=self.context.region), ) Tags.of(scope=cast(IConstruct, self)).add( key="Env", value=f"orbit-{self.context.name}") if self.context.networking.vpc_id is None: raise ValueError("self.context.networking.vpc_id is None.") if self.context.networking.availability_zones is None: raise ValueError( "self.context.networking.availability_zones is None.") self.i_vpc = ec2.Vpc.from_vpc_attributes( scope=self, id="vpc", vpc_id=self.context.networking.vpc_id, availability_zones=self.context.networking.availability_zones, ) self.role_eks_cluster = self._create_role_cluster() self.role_eks_env_nodegroup = self._create_env_nodegroup_role() self.role_fargate_profile = self._create_role_fargate_profile() self.role_cluster_autoscaler = self._create_cluster_autoscaler_role() if self.context.user_pool_id: self.context.cognito_users_url = orbit_cognito.get_users_url( user_pool_id=self.context.user_pool_id, region=self.context.region) cognito_pool_arn: str = orbit_cognito.get_pool_arn( user_pool_id=self.context.user_pool_id, region=self.context.region, account=self.context.account_id) self.user_pool: cognito.UserPool = self._get_user_pool( user_pool_arn=cognito_pool_arn) else: raise Exception("Missing Cognito User Pool ID ('user_pool_id') ") self.user_pool_client = self._create_user_pool_client() self.identity_pool = self._create_identity_pool() self.token_validation_lambda = self._create_token_validation_lambda() self.eks_service_lambda = self._create_eks_service_lambda() self.cluster_pod_security_group = self._create_cluster_pod_security_group( ) self.context_parameter = self._create_manifest_parameter() self._create_post_authentication_lambda()
def __init__(self, scope: Construct, id: str, *, deployment: Deployment, policy: Policy, cluster: ICluster, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) policy.add_stack(self) if deployment == Deployment.PRODUCTION: desired_count = 2 priority = 62 else: desired_count = 1 priority = 162 api_fqdn = dns.subdomain_to_fqdn("api.master") api_url = f"https://{api_fqdn}" sentry_dsn = parameter_store.add_secure_string( f"/MasterServerWeb/{deployment.value}/SentryDSN").parameter ECSHTTPSContainer( self, self.application_name, subdomain_name=self.subdomain_name, deployment=deployment, policy=policy, application_name=self.application_name, image_name="ghcr.io/openttd/master-server-web", port=80, memory_limit_mib=96, desired_count=desired_count, cluster=cluster, priority=priority, command=[ "--api-url", api_url, "run", "-p", "80", "-h", "0.0.0.0", ], environment={ "WEBCLIENT_SENTRY_ENVIRONMENT": deployment.value.lower(), }, secrets={ "WEBCLIENT_SENTRY_DSN": Secret.from_ssm_parameter(sentry_dsn), }, )
def __init__(self, scope: Construct, id: str, vpc: IVpc, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Stack", "Common-Ecs") self._cluster = Cluster( self, "Cluster", vpc=vpc, ) asg = AutoScalingGroup( self, "ClusterASG", vpc=vpc, instance_type=InstanceType("t3a.small"), machine_image=EcsOptimizedImage.amazon_linux2(), min_capacity=4, ) self._cluster.add_auto_scaling_group(asg) # Create a SecurityGroup that the NLB can use to allow traffic from # NLB to us. This avoids a cyclic dependency. self.security_group = SecurityGroup( self, "SecurityGroup", vpc=vpc, allow_all_outbound=False, ) # Only use "source_security_group" to check if flows come from ECS. # Do not use it to allow traffic in ECS; use "security_group" for # that. assert isinstance(asg.node.children[0], SecurityGroup) self.source_security_group = asg.node.children[0] # We could also make an additional security-group and add that to # the ASG, but it keeps adding up. This makes it a tiny bit # easier to get an overview what traffic is allowed from the # console on AWS. asg.node.children[0].add_ingress_rule( peer=self.security_group, connection=Port.tcp_range(32768, 65535), description="NLB-self to target", ) asg.node.children[0].add_ingress_rule( peer=self.security_group, connection=Port.udp_range(32768, 65535), description="NLB-self to target (UDP)", )
def __init__(self, scope: Construct, id: str, context: "Context", parameters: Dict[str, Any]) -> None: super().__init__( scope=scope, id=id, stack_name=id, env=Environment(account=context.account_id, region=context.region), ) Tags.of(scope=cast(IConstruct, self)).add( key="Env", value=f"orbit-{context.name}") _logger.debug( "Passing Plugin CDK Stack parameters to the Nested stack Cfn parameters" ) NestedCfnStack(self, id="custom-cfn-stack", parameters=parameters)
def add_nlb(self, scope: Construct, service: IEc2Service, port: Port, subdomain_name: str, description: str) -> None: port_dict = port.to_rule_json() Tags.of(service).add("NLB-protocol", port_dict["ipProtocol"]) Tags.of(service).add("NLB-port", str(port_dict["fromPort"])) self.create_alias(scope, subdomain_name) self.security_group.add_ingress_rule( peer=Peer.any_ipv6(), connection=port, description=f"{description} (IPv6)") self.security_group.add_ingress_rule( peer=Peer.any_ipv4(), connection=port, description=f"{description} (IPv4)")
def __init__(self, scope: Construct, id: str, context: "Context", team_context: "TeamContext", parameters: Dict[str, Any]) -> None: super().__init__( scope=scope, id=id, stack_name=id, env=Environment(account=context.account_id, region=context.region), ) Tags.of(scope=cast(IConstruct, self)).add( key="Env", value=f"orbit-{context.name}") # Collecting required parameters team_space_props: Dict[str, Any] = { "account_id": context.account_id, "region": context.region, "partition": core.Aws.PARTITION, "env_name": context.name, "teamspace_name": team_context.name, "lake_role_name": f"orbit-{context.name}-{team_context.name}-role", "vpc_id": context.networking.vpc_id, "subnet_ids": [ s.subnet_id for s in context.networking.public_subnets + context.networking.isolated_subnets + context.networking.private_subnets ], "team_security_group_id": team_context.team_security_group_id, "team_kms_key_arn": team_context.team_kms_key_arn, } self._redshift_clusters = RedshiftClusters( self, id="redshift-clusters-for-teamspace", team_space_props=team_space_props, plugin_params=parameters, )
def __init__( self, scope: Construct, id: str, *, deployment: Deployment, additional_fqdns: Optional[List[str]] = None, **kwargs, ) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) func = lambda_edge.create_function( self, f"BananasCdnRedirect{deployment.value}", runtime=Runtime.NODEJS_10_X, handler="index.handler", code=Code.from_asset("./lambdas/bananas-cdn"), ) s3_cloud_front = S3CloudFront( self, "S3CloudFront", subdomain_name=self.subdomain_name, error_folder="/errors", lambda_function_associations=[ LambdaFunctionAssociation( event_type=LambdaEdgeEventType.ORIGIN_REQUEST, lambda_function=func, ), ], price_class=PriceClass.PRICE_CLASS_ALL, additional_fqdns=additional_fqdns, viewer_protocol_policy=ViewerProtocolPolicy. ALLOW_ALL, # OpenTTD client doesn't support HTTPS ) self.bucket = s3_cloud_front.bucket_site S3CloudFrontPolicy( self, "S3cloudFrontPolicy", s3_cloud_front=s3_cloud_front, with_s3_get_object_access=True, )
def __init__(self, scope: Construct, stack_id: str) -> None: super().__init__(scope, stack_id) ############################################################################################ # ### DATASET STAGING S3 BUCKET ############################################################ ############################################################################################ self.staging_bucket = aws_s3.Bucket( self, "dataset-staging-bucket", bucket_name=ResourceName.STAGING_BUCKET_NAME.value, access_control=aws_s3.BucketAccessControl.PRIVATE, block_public_access=aws_s3.BlockPublicAccess.BLOCK_ALL, versioned=True, removal_policy=RemovalPolicy.DESTROY, ) Tags.of(self).add("ApplicationLayer", "staging") # type: ignore[arg-type]
def __init__(self, scope: Construct, stack_id: str, *, env_name: str, storage_bucket: aws_s3.Bucket) -> None: super().__init__(scope, stack_id) account_principal = aws_iam.AccountPrincipal(account_id=276514628126) external_id = { PRODUCTION_ENVIRONMENT_NAME: "koordinates-jAddR" }.get(env_name, "koordinates-4BnJQ") role = aws_iam.Role( self, "koordinates-read-role", role_name=f"koordinates-s3-access-read-{env_name}", assumed_by=account_principal, # type: ignore[arg-type] external_id=external_id, max_session_duration=MAX_SESSION_DURATION, ) storage_bucket.grant_read(role) # type: ignore[arg-type] Tags.of(self).add("ApplicationLayer", "lds") # type: ignore[arg-type]
def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add('landing_zone', self.zone_name) self.networking = NetworkingLayer( self, self.zone_name, cidr=self.cidr_block, subnet_configuration=self.subnet_configuration) self.backup_policy = BackupStrategyConstruct(self, 'Backup', landing_zone=self) self.security_group = ec2.SecurityGroup( self, 'SecurityGroup', description='Default-SG for {} landing zone'.format( self.zone_name), vpc=self.vpc, allow_all_outbound=True) self.security_group.add_ingress_rule( peer=ec2.Peer.any_ipv4(), connection=ec2.Port.all_icmp(), description='Grant icmp from anywhere') for address in ('72.90.160.65/32', '10.0.0.0/8', '192.168.0.0/16'): self.security_group.add_ingress_rule( peer=ec2.Peer.ipv4(address), connection=ec2.Port.all_traffic(), description='Grant any from ' + address) self.security_group.add_ingress_rule( peer=ec2.Peer.ipv4(address), connection=ec2.Port.tcp(3389), description='Grant rdp from ' + address) self.security_group.add_ingress_rule( peer=ec2.Peer.ipv4(address), connection=ec2.Port.tcp(22), description='Grant ssh from ' + address)
def __init__(self, scope: Construct, id: str, *, deployment: Deployment, additional_fqdns: Optional[List[str]] = None, **kwargs) -> None: super().__init__(scope, id, **kwargs) Tags.of(self).add("Application", self.application_name) Tags.of(self).add("Deployment", deployment.value) func = lambda_edge.create_function( self, "CdnIndexRedirect", runtime=Runtime.NODEJS_10_X, handler="index.handler", code=Code.from_asset("./lambdas/index-redirect"), ) s3_cloud_front = S3CloudFront( self, "S3CloudFront", subdomain_name=self.subdomain_name, error_folder="/errors", lambda_function_associations=[ LambdaFunctionAssociation( event_type=LambdaEdgeEventType.ORIGIN_REQUEST, lambda_function=func, ), ], additional_fqdns=additional_fqdns, price_class=PriceClass.PRICE_CLASS_ALL, ) S3CloudFrontPolicy( self, "S3cloudFrontPolicy", s3_cloud_front=s3_cloud_front, with_s3_get_object_access=True, )
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) vpc = ec2.Vpc( self, "Adam_MyVpc", cidr="10.13.0.0/21", max_azs=2, nat_gateways=1, subnet_configuration=[ ec2.SubnetConfiguration(name="public", cidr_mask=24, subnet_type=ec2.SubnetType.PUBLIC), ec2.SubnetConfiguration(name="private", cidr_mask=24, subnet_type=ec2.SubnetType.PRIVATE), ec2.SubnetConfiguration(name="isolated", cidr_mask=24, subnet_type=ec2.SubnetType.ISOLATED) ]) Tags.of(vpc).add("Owner", "Adam")
def __init__(self, scope: Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) global g_lambda_edge Tags.of(self).add("Stack", "Common-Lambda-Edge") self._role = Role( self, "EdgeLambdaRole", assumed_by=CompositePrincipal( ServicePrincipal("lambda.amazonaws.com"), ServicePrincipal("edgelambda.amazonaws.com"), ), managed_policies=[ ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole"), ], ) if g_lambda_edge is not None: raise Exception("Only a single LambdaEdgeStack instance can exist") g_lambda_edge = self
def __init__( self, scope: Construct, stack_id: str, *, botocore_lambda_layer: aws_lambda_python.PythonLayerVersion, env_name: str, storage_bucket: aws_s3.Bucket, validation_results_table: Table, ) -> None: # pylint: disable=too-many-locals, too-many-statements super().__init__(scope, stack_id) ############################################################################################ # PROCESSING ASSETS TABLE processing_assets_table = Table( self, f"{env_name}-processing-assets", env_name=env_name, parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME, sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING), ) ############################################################################################ # BATCH JOB DEPENDENCIES batch_job_queue = BatchJobQueue( self, "batch-job-queue", env_name=env_name, processing_assets_table=processing_assets_table, ).job_queue s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name( "AmazonS3ReadOnlyAccess" ) ############################################################################################ # UPDATE CATALOG UPDATE MESSAGE QUEUE dead_letter_queue = aws_sqs.Queue( self, "dead-letter-queue", visibility_timeout=LAMBDA_TIMEOUT, ) self.message_queue = aws_sqs.Queue( self, "update-catalog-message-queue", visibility_timeout=LAMBDA_TIMEOUT, dead_letter_queue=aws_sqs.DeadLetterQueue(max_receive_count=3, queue=dead_letter_queue), ) self.message_queue_name_parameter = aws_ssm.StringParameter( self, "update-catalog-message-queue-name", string_value=self.message_queue.queue_name, description=f"Update Catalog Message Queue Name for {env_name}", parameter_name=ParameterName.UPDATE_CATALOG_MESSAGE_QUEUE_NAME.value, ) populate_catalog_lambda = BundledLambdaFunction( self, "populate-catalog-bundled-lambda-function", directory="populate_catalog", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, botocore_lambda_layer=botocore_lambda_layer, ) self.message_queue.grant_consume_messages(populate_catalog_lambda) populate_catalog_lambda.add_event_source( SqsEventSource(self.message_queue, batch_size=1) # type: ignore[arg-type] ) ############################################################################################ # STATE MACHINE TASKS check_stac_metadata_task = LambdaTask( self, "check-stac-metadata-task", directory="check_stac_metadata", botocore_lambda_layer=botocore_lambda_layer, extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) assert check_stac_metadata_task.lambda_function.role check_stac_metadata_task.lambda_function.role.add_managed_policy( policy=s3_read_only_access_policy ) for table in [processing_assets_table, validation_results_table]: table.grant_read_write_data(check_stac_metadata_task.lambda_function) table.grant( check_stac_metadata_task.lambda_function, "dynamodb:DescribeTable", ) content_iterator_task = LambdaTask( self, "content-iterator-task", directory="content_iterator", botocore_lambda_layer=botocore_lambda_layer, result_path=f"$.{CONTENT_KEY}", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) check_files_checksums_directory = "check_files_checksums" check_files_checksums_default_payload_object = { f"{DATASET_ID_KEY}.$": f"$.{DATASET_ID_KEY}", f"{VERSION_ID_KEY}.$": f"$.{VERSION_ID_KEY}", f"{METADATA_URL_KEY}.$": f"$.{METADATA_URL_KEY}", f"{FIRST_ITEM_KEY}.$": f"$.{CONTENT_KEY}.{FIRST_ITEM_KEY}", f"{ASSETS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{ASSETS_TABLE_NAME_KEY}", f"{RESULTS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{RESULTS_TABLE_NAME_KEY}", } check_files_checksums_single_task = BatchSubmitJobTask( self, "check-files-checksums-single-task", env_name=env_name, directory=check_files_checksums_directory, s3_policy=s3_read_only_access_policy, job_queue=batch_job_queue, payload_object=check_files_checksums_default_payload_object, container_overrides_command=[ "--dataset-id", f"Ref::{DATASET_ID_KEY}", "--version-id", f"Ref::{VERSION_ID_KEY}", "--first-item", f"Ref::{FIRST_ITEM_KEY}", "--assets-table-name", f"Ref::{ASSETS_TABLE_NAME_KEY}", "--results-table-name", f"Ref::{RESULTS_TABLE_NAME_KEY}", ], ) array_size = int( aws_stepfunctions.JsonPath.number_at(f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}") ) check_files_checksums_array_task = BatchSubmitJobTask( self, "check-files-checksums-array-task", env_name=env_name, directory=check_files_checksums_directory, s3_policy=s3_read_only_access_policy, job_queue=batch_job_queue, payload_object=check_files_checksums_default_payload_object, container_overrides_command=[ "--dataset-id", f"Ref::{DATASET_ID_KEY}", "--version-id", f"Ref::{VERSION_ID_KEY}", "--first-item", f"Ref::{FIRST_ITEM_KEY}", "--assets-table-name", f"Ref::{ASSETS_TABLE_NAME_KEY}", "--results-table-name", f"Ref::{RESULTS_TABLE_NAME_KEY}", ], array_size=array_size, ) for reader in [ content_iterator_task.lambda_function, check_files_checksums_single_task.job_role, check_files_checksums_array_task.job_role, ]: processing_assets_table.grant_read_data(reader) # type: ignore[arg-type] processing_assets_table.grant( reader, "dynamodb:DescribeTable" # type: ignore[arg-type] ) for writer in [ check_files_checksums_single_task.job_role, check_files_checksums_array_task.job_role, ]: validation_results_table.grant_read_write_data(writer) # type: ignore[arg-type] validation_results_table.grant( writer, "dynamodb:DescribeTable" # type: ignore[arg-type] ) validation_summary_task = LambdaTask( self, "validation-summary-task", directory="validation_summary", botocore_lambda_layer=botocore_lambda_layer, result_path=f"$.{VALIDATION_KEY}", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) validation_results_table.grant_read_data(validation_summary_task.lambda_function) validation_results_table.grant( validation_summary_task.lambda_function, "dynamodb:DescribeTable" ) import_dataset_role = aws_iam.Role( self, "import-dataset", assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type] "batchoperations.s3.amazonaws.com" ), ) import_asset_file_function = ImportFileFunction( self, directory="import_asset_file", invoker=import_dataset_role, env_name=env_name, botocore_lambda_layer=botocore_lambda_layer, ) import_metadata_file_function = ImportFileFunction( self, directory="import_metadata_file", invoker=import_dataset_role, env_name=env_name, botocore_lambda_layer=botocore_lambda_layer, ) import_dataset_task = LambdaTask( self, "import-dataset-task", directory="import_dataset", botocore_lambda_layer=botocore_lambda_layer, result_path=f"$.{IMPORT_DATASET_KEY}", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) import_dataset_task.lambda_function.add_to_role_policy( aws_iam.PolicyStatement( resources=[import_dataset_role.role_arn], actions=["iam:PassRole"], ), ) import_dataset_task.lambda_function.add_to_role_policy( aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"]) ) for table in [processing_assets_table]: table.grant_read_data(import_dataset_task.lambda_function) table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable") # Import status check wait_before_upload_status_check = Wait( self, "wait-before-upload-status-check", time=WaitTime.duration(Duration.seconds(10)), ) upload_status_task = LambdaTask( self, "upload-status", directory="upload_status", botocore_lambda_layer=botocore_lambda_layer, result_path="$.upload_status", extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) validation_results_table.grant_read_data(upload_status_task.lambda_function) validation_results_table.grant(upload_status_task.lambda_function, "dynamodb:DescribeTable") upload_status_task.lambda_function.add_to_role_policy(ALLOW_DESCRIBE_ANY_S3_JOB) # Parameters import_asset_file_function_arn_parameter = aws_ssm.StringParameter( self, "import asset file function arn", string_value=import_asset_file_function.function_arn, description=f"Import asset file function ARN for {env_name}", parameter_name=ParameterName.PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value, ) import_metadata_file_function_arn_parameter = aws_ssm.StringParameter( self, "import metadata file function arn", string_value=import_metadata_file_function.function_arn, description=f"Import metadata file function ARN for {env_name}", parameter_name=ParameterName.PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value, ) import_dataset_role_arn_parameter = aws_ssm.StringParameter( self, "import dataset role arn", string_value=import_dataset_role.role_arn, description=f"Import dataset role ARN for {env_name}", parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.value, ) update_dataset_catalog = LambdaTask( self, "update-dataset-catalog", directory="update_dataset_catalog", botocore_lambda_layer=botocore_lambda_layer, extra_environment={ENV_NAME_VARIABLE_NAME: env_name}, ) self.message_queue.grant_send_messages(update_dataset_catalog.lambda_function) for storage_writer in [ import_dataset_role, import_dataset_task.lambda_function, import_asset_file_function, import_metadata_file_function, populate_catalog_lambda, update_dataset_catalog.lambda_function, ]: storage_bucket.grant_read_write(storage_writer) # type: ignore[arg-type] grant_parameter_read_access( { import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function], import_dataset_role_arn_parameter: [import_dataset_task.lambda_function], import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function], processing_assets_table.name_parameter: [ check_stac_metadata_task.lambda_function, content_iterator_task.lambda_function, import_dataset_task.lambda_function, ], validation_results_table.name_parameter: [ check_stac_metadata_task.lambda_function, content_iterator_task.lambda_function, validation_summary_task.lambda_function, upload_status_task.lambda_function, ], self.message_queue_name_parameter: [update_dataset_catalog.lambda_function], } ) success_task = aws_stepfunctions.Succeed(self, "success") upload_failure = aws_stepfunctions.Fail(self, "upload failure") validation_failure = aws_stepfunctions.Succeed(self, "validation failure") ############################################################################################ # STATE MACHINE dataset_version_creation_definition = ( check_stac_metadata_task.next(content_iterator_task) .next( aws_stepfunctions.Choice( # type: ignore[arg-type] self, "check_files_checksums_maybe_array" ) .when( aws_stepfunctions.Condition.number_equals( f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}", 1 ), check_files_checksums_single_task.batch_submit_job, ) .otherwise(check_files_checksums_array_task.batch_submit_job) .afterwards() ) .next( aws_stepfunctions.Choice(self, "content_iteration_finished") .when( aws_stepfunctions.Condition.number_equals( f"$.{CONTENT_KEY}.{NEXT_ITEM_KEY}", -1 ), validation_summary_task.next( aws_stepfunctions.Choice( # type: ignore[arg-type] self, "validation_successful" ) .when( aws_stepfunctions.Condition.boolean_equals( f"$.{VALIDATION_KEY}.{SUCCESS_KEY}", True ), import_dataset_task.next( wait_before_upload_status_check # type: ignore[arg-type] ) .next(upload_status_task) .next( aws_stepfunctions.Choice( self, "import_completed" # type: ignore[arg-type] ) .when( aws_stepfunctions.Condition.and_( aws_stepfunctions.Condition.string_equals( f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Complete" ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{METADATA_UPLOAD_KEY}.status", "Complete", ), ), update_dataset_catalog.next( success_task # type: ignore[arg-type] ), ) .when( aws_stepfunctions.Condition.or_( aws_stepfunctions.Condition.string_equals( f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Cancelled", ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Failed" ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{METADATA_UPLOAD_KEY}.status", "Cancelled", ), aws_stepfunctions.Condition.string_equals( f"$.upload_status.{METADATA_UPLOAD_KEY}.status", "Failed", ), ), upload_failure, # type: ignore[arg-type] ) .otherwise( wait_before_upload_status_check # type: ignore[arg-type] ) ), ) .otherwise(validation_failure) # type: ignore[arg-type] ), ) .otherwise(content_iterator_task) ) ) self.state_machine = aws_stepfunctions.StateMachine( self, f"{env_name}-dataset-version-creation", definition=dataset_version_creation_definition, # type: ignore[arg-type] ) self.state_machine_parameter = aws_ssm.StringParameter( self, "state machine arn", description=f"State machine ARN for {env_name}", parameter_name=ParameterName.PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long string_value=self.state_machine.state_machine_arn, ) Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]
def __init__(self, scope: Construct, stack_id: str, *, env_name: str) -> None: super().__init__(scope, stack_id) ############################################################################################ # ### DEPLOYMENT VERSION ################################################################### ############################################################################################ aws_ssm.StringParameter( self, "git-branch", parameter_name=f"/{env_name}/git_branch", string_value=GIT_BRANCH, description="Deployment git branch", ) aws_ssm.StringParameter( self, "git-commit", parameter_name=f"/{env_name}/git_commit", string_value=GIT_COMMIT, description="Deployment git commit", ) aws_ssm.StringParameter( self, "git-tag", parameter_name=f"/{env_name}/version", string_value=GIT_TAG, description="Deployment version", ) ############################################################################################ # ### STORAGE S3 BUCKET #################################################################### ############################################################################################ self.storage_bucket = aws_s3.Bucket( self, "storage-bucket", bucket_name=ResourceName.STORAGE_BUCKET_NAME.value, access_control=aws_s3.BucketAccessControl.PRIVATE, block_public_access=aws_s3.BlockPublicAccess.BLOCK_ALL, versioned=True, removal_policy=REMOVAL_POLICY, ) ############################################################################################ # ### APPLICATION DB ####################################################################### ############################################################################################ self.datasets_table = Table( self, f"{env_name}-datasets", env_name=env_name, parameter_name=ParameterName.STORAGE_DATASETS_TABLE_NAME, ) self.datasets_table.add_global_secondary_index( index_name=DatasetsTitleIdx.Meta.index_name, partition_key=aws_dynamodb.Attribute( name="title", type=aws_dynamodb.AttributeType.STRING ), ) self.validation_results_table = Table( self, f"{env_name}-validation-results", env_name=env_name, parameter_name=ParameterName.STORAGE_VALIDATION_RESULTS_TABLE_NAME, sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING), ) self.validation_results_table.add_global_secondary_index( index_name=ValidationOutcomeIdx.Meta.index_name, partition_key=aws_dynamodb.Attribute( name=ValidationOutcomeIdx.pk.attr_name, type=aws_dynamodb.AttributeType.STRING ), sort_key=aws_dynamodb.Attribute( name=ValidationOutcomeIdx.result.attr_name, type=aws_dynamodb.AttributeType.STRING ), ) Tags.of(self).add("ApplicationLayer", "storage") # type: ignore[arg-type]