def __init__(self, scope: core.Construct, construct_id: str, queue_context: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) q = dict(self.node.try_get_context(queue_context)) queue_dlq = _sqs.Queue(self, q["queue_dlq_name"], queue_name=q["queue_dlq_name"]) queue = _sqs.Queue( self, q["queue_name"], queue_name=q["queue_name"], dead_letter_queue=_sqs.DeadLetterQueue( max_receive_count=q["queue_dlq_max_receive_count"], queue=queue_dlq), encryption=_sqs.QueueEncryption.KMS_MANAGED, visibility_timeout=Duration.seconds(30), delivery_delay=Duration.seconds(15), retention_period=Duration.hours(14), ) self.queue = queue self.queue_dlq = queue_dlq # Outputs core.CfnOutput(self, "QueueUrl", value=queue.queue_url)
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) stream = kds.Stream(self, "InputStream", shard_count=1, retention_period=Duration.hours(24)) self._stream = stream
def __init__(self, scope: Construct, id: str, functions: LambdaLib, **kwargs) -> None: super().__init__(scope, id) # Step Function submit_job = tasks.LambdaInvoke(self, "Submit Job", lambda_function=functions.send_email_approval, payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}), result_path=sfn.JsonPath.DISCARD ) wait_x = sfn.Wait(self, "Wait", time= sfn.WaitTime.duration(Duration.minutes(2)) ) get_status = tasks.LambdaInvoke(self, "Get Job Status", lambda_function=functions.check_status_dynamo, payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}), result_path="$.status" ) restrict_es = tasks.LambdaInvoke(self, "Restric ES Policy", lambda_function=functions.restric_es_policy, payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}), ) restrict_rds = tasks.LambdaInvoke(self, "Restric RDS", lambda_function=functions.restric_rds_policy, payload=sfn.TaskInput.from_object({'ExecutionContext.$': '$$'}), ) restrict_es_condition = sfn.Condition.string_equals("$.detail.additionalEventData.configRuleName", constants.CONFIG_RULE_ES_PUBLIC) restrict_rds_condition = sfn.Condition.string_equals("$.detail.additionalEventData.configRuleName", constants.CONFIG_RULE_RDS_PUBLIC) definition = (submit_job.next(wait_x) .next(get_status) .next(sfn.Choice(self, "Job Complete?") .when(sfn.Condition.string_equals("$.status.Payload.status", "Rejected!"), wait_x) # .when(sfn.Condition.string_equals("$.status.Payload.status", "NON_COMPLIANT"), final_task) # .when(sfn.Condition.string_equals("$.status.Payload.status", "Accepted!"), final_task)) .otherwise(sfn.Choice(self, "Remediation Choice") .when(restrict_es_condition, restrict_es) .when(restrict_rds_condition, restrict_rds))) ) self.state_machine = sfn.StateMachine(self, "StateMachine", definition=definition, timeout=Duration.hours(2) )
def base_kinesis_stream(construct, **kwargs): """ Function that generates a Kinesis Data Stream. :param construct: Custom construct that will use this function. From the external construct is usually 'self'. :param kwargs: Consist of required 'stream_name'. :return: Kinesis Stream Construct. """ stream_name = construct.prefix + "_" + kwargs[ "stream_name"] + "_" + "stream" + "_" + construct.environment_ stream_retention_period = Duration.hours( kwargs["retention_period"]) if kwargs.get( "retention_period") is not None else None kinesis_stream = stream.Stream( construct, id=stream_name, stream_name=stream_name, shard_count=kwargs["shard_count"], retention_period=stream_retention_period, ) return kinesis_stream
def add_low_efs_burst_credit_alarms(self, filesystem: FileSystem, email_address: str) -> None: ''' Set up CloudWatch Alarms that will warn when the given filesystem's burst credits are below four different thresholds. We send an email to the given address when an Alarm breaches. ''' # Set up the SNS Topic that will send the emails. # ==================== # 1) KMS key to use to encrypt events within the SNS Topic. The Key is optional key = Key( self, 'SNSEncryptionKey', description= 'Used to encrypt the SNS Topic for sending EFS Burst Credit alerts', enable_key_rotation=True, removal_policy=RemovalPolicy.DESTROY, trust_account_identities=True) key.grant(ServicePrincipal('cloudwatch.amazonaws.com'), 'kms:Decrypt', 'kms:GenerateDataKey') # 2) SNS Topic that will be alerted by CloudWatch and will send the email in response. sns_topic = Topic(self, 'BurstAlertEmailTopic', master_key=key) sns_topic.grant_publish(ServicePrincipal('cloudwatch.amazonaws.com')) sns_topic.add_subscription(EmailSubscription(email_address)) alarm_action = SnsAction(sns_topic) # Set up the CloudWatch Alarm(s) and have them trigger SNS events when breached. # ====================== # 1) CDK helper to define the CloudWatch Metric that we're interested in. burst_credits_metric = Metric( metric_name='BurstCreditBalance', namespace='AWS/EFS', dimensions={"FileSystemId": filesystem.file_system_id}, # One 99-th percentile data point sample every hour period=Duration.hours(1), statistic='p99') # 2) Create the alarms thresholds = [ { "id": 'CAUTION-EfsBurstCredits', "name": f"CAUTION Burst Credits - {filesystem.file_system_id}", "threshold": int(2.00 * 2**40), "message": f"CAUTION. 2 TiB Threshold Breached: EFS {filesystem.file_system_id} is depleting burst credits. Add data to the EFS to increase baseline throughput.", # Alarm after 6 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 6hrs "datapoints": 6 }, { "id": 'WARNING-EfsBurstCredits', "name": f"WARNING Burst Credits - {filesystem.file_system_id}", "threshold": int(1.25 * 2**40), "message": f"WARNING. 1.25 TiB Threshold Breached: EFS {filesystem.file_system_id} is depleting burst credits. Add data to the EFS to increase baseline throughput.", # Alarm after 6 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 6hrs "datapoints": 6 }, { "id": 'ALERT-EfsBurstCredits', "name": f"ALERT Burst Credits - {filesystem.file_system_id}", "threshold": int(0.50 * 2**40), "message": f"ALERT! 500 GiB Threshold Breached: EFS {filesystem.file_system_id} is running out of burst credits. Add data to the EFS to increase baseline throughput or else the Render Farm may cease operation.", # Alarm after 6 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 6hrs "datapoints": 6 }, { "id": 'EMERGENCY-EfsBurstCredits', "name": f"EMERGENCY Burst Credits - {filesystem.file_system_id}", "threshold": int(0.10 * 2**40), "message": f"EMERGENCY! 100 GiB Threshold Breached: EFS {filesystem.file_system_id} is running out of burst credits. Add data to the EFS to increase baseline throughput or else the Render Farm will cease operation.", # Alarm after 2 datapoints below threshold. We have 1 datapoint every hour. So, we alarm if below threshold for 2hrs "datapoints": 2 }, ] for config in thresholds: alarm = burst_credits_metric.create_alarm( self, config['id'], alarm_name=config['name'], actions_enabled=True, alarm_description=config['message'], treat_missing_data=TreatMissingData.NOT_BREACHING, threshold=config['threshold'], comparison_operator=ComparisonOperator.LESS_THAN_THRESHOLD, evaluation_periods=config['datapoints']) alarm.add_alarm_action(alarm_action)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # TODO: add resources for Part 2, 3 of blog post. # Kinesis Data Streams kds = kinesis.Stream(self, "KinesisTweets", stream_name="kinesis-tweets", shard_count=5, retention_period=Duration.hours(48)) # Fargate Task Role task_role = iam.Role( self, 'task_role', assumed_by=iam.ServicePrincipal('ecs-tasks.amazonaws.com')) # Policy to allow task to put records into Kinessis task_role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'kinesis:PutRecord', 'kinesis:PutRecords', 'kinesis:DescribeStream' ], resources=[kds.stream_arn])) # Policy to get secret from SecretsManager task_role.add_to_policy( iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'secretsmanager:GetResourcePolicy', 'secretsmanager:GetSecretValue', 'secretsmanager:DescribeSecret', 'secretsmanager:ListSecretVersionIds' ], resources=['*'])) # VPC vpc = ec2.Vpc( self, 'FargateVPC', max_azs=2 # Default is all AZs in the region ) # ECS Cluster cluster = ecs.Cluster(self, 'EcsCluster', vpc=vpc) # Fargate Task Definition task_definition = ecs.FargateTaskDefinition(self, 'ServiceTaskDefinition', cpu=256, memory_limit_mib=512, task_role=task_role) # Fargate log driver fargate_logger = ecs.AwsLogDriver(stream_prefix='fargate_twitter_logs') # Container task_definition.add_container( 'ServiceContainer', image=ecs.ContainerImage.from_asset('./ECSContainerFiles'), environment={ 'KINESIS_STREAM_NAME': kds.stream_name, 'REGION_NAME': self.region, 'KEYWORD': 'trump', 'SECRETS_NAME': 'TwitterAPISecrets' }, logging=fargate_logger) # Fargate Service service = ecs.FargateService(self, 'ServiceFargateService', task_definition=task_definition, assign_public_ip=True, cluster=cluster)
from aws_cdk.core import Duration MAX_SESSION_DURATION = Duration.hours(12)
def __init__( self, scope: Construct, id: str, context: "Context", team_name: str, team_policies: List[str], image: Optional[str], ) -> None: self.scope = scope self.id = id self.context: "Context" = context self.team_name: str = team_name self.team_policies: List[str] = team_policies self.image: Optional[str] = image super().__init__( scope=scope, id=id, stack_name=id, env=Environment(account=self.context.account_id, region=self.context.region), ) Tags.of(scope=cast(IConstruct, self)).add( key="Env", value=f"orbit-{self.context.name}") Tags.of(scope=cast(IConstruct, self)).add(key="TeamSpace", value=self.team_name) if self.context.networking.vpc_id is None: raise ValueError("self.context.networking.vpc_id is None!") self.i_vpc = ec2.Vpc.from_vpc_attributes( scope=self, id="vpc", vpc_id=self.context.networking.vpc_id, availability_zones=cast( List[str], self.context.networking.availability_zones), ) self.i_isolated_subnets = Ec2Builder.build_subnets( scope=self, subnet_manifests=context.networking.isolated_subnets) self.i_private_subnets = Ec2Builder.build_subnets( scope=self, subnet_manifests=context.networking.private_subnets) administrator_arns: List[str] = [ ] # A place to add other admins if needed for KMS admin_principals = iam.CompositePrincipal( *[iam.ArnPrincipal(arn) for arn in administrator_arns], iam.ArnPrincipal(f"arn:aws:iam::{self.context.account_id}:root"), ) self.team_kms_key: kms.Key = kms.Key( self, id="kms-key", removal_policy=core.RemovalPolicy.RETAIN, enabled=True, enable_key_rotation=True, policy=iam.PolicyDocument(statements=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=["kms:*"], resources=["*"], principals=[cast(iam.IPrincipal, admin_principals)], ) ]), ) self.team_security_group: ec2.SecurityGroup = Ec2Builder.build_team_security_group( scope=self, context=context, team_name=self.team_name, vpc=self.i_vpc) self.policies: List[str] = self.team_policies if self.context.scratch_bucket_arn: self.scratch_bucket: s3.Bucket = cast( s3.Bucket, s3.Bucket.from_bucket_attributes( scope=self, id="scratch_bucket", bucket_arn=self.context.scratch_bucket_arn, bucket_name=self.context.scratch_bucket_arn.split(":::") [1], ), ) else: raise Exception( "Scratch bucket was not provided in Manifest ('ScratchBucketArn')" ) self.role_eks_pod = IamBuilder.build_team_role( scope=self, context=self.context, team_name=self.team_name, policy_names=self.policies, scratch_bucket=cast(s3.IBucket, self.scratch_bucket), team_kms_key=self.team_kms_key, session_timeout=Duration.hours(12), ) shared_fs_name: str = f"orbit-{context.name}-{self.team_name}-shared-fs" if context.shared_efs_fs_id is None: raise Exception( "Shared EFS File system ID was not provided in Manifest ('SharedEfsFsId')" ) if context.shared_efs_sg_id is None: raise Exception( "Shared EFS File system security group ID was not provided in Manifest ('SharedEfsSgId')" ) self.shared_fs: efs.FileSystem = cast( efs.FileSystem, efs.FileSystem.from_file_system_attributes( scope=self, id=shared_fs_name, file_system_id=context.shared_efs_fs_id, security_group=ec2.SecurityGroup.from_security_group_id( scope=self, id="team_sec_group", security_group_id=context.shared_efs_sg_id), ), ) self.efs_ap: efs.AccessPoint = EfsBuilder.build_file_system_access_point( scope=self, team_name=team_name, shared_fs=self.shared_fs, path="shared", ap_name=f"{team_name}-shared") self.efs_ap_team_private: efs.AccessPoint = EfsBuilder.build_file_system_access_point( scope=self, team_name=team_name, shared_fs=self.shared_fs, path="team", ap_name=f"{team_name}-private") team_ssm_parameter_name: str = f"/orbit/{context.name}/teams/{self.team_name}/team" self.context_parameter: ssm.StringParameter = ssm.StringParameter( scope=self, id=team_ssm_parameter_name, string_value=json.dumps({ "EfsId": self.shared_fs.file_system_id, "EfsApId": self.efs_ap.access_point_id, "EfsApIdPrivate": self.efs_ap_team_private.access_point_id, "EksPodRoleArn": self.role_eks_pod.role_arn, "ScratchBucket": self.scratch_bucket.bucket_name, "TeamKmsKeyArn": self.team_kms_key.key_arn, "TeamSecurityGroupId": self.team_security_group.security_group_id, }), type=ssm.ParameterType.STRING, description="Orbit Workbench Team Context.", parameter_name=team_ssm_parameter_name, simple_name=False, tier=ssm.ParameterTier.INTELLIGENT_TIERING, )