def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Create the queue MySqsQueue = sqs.Queue(self, "MySqsQueue") # Create the Topic MySnsTopic = sns.Topic(self, "MySnsTopic") # Create an SQS topic subscription object sqsSubscription = snssubs.SqsSubscription(MySqsQueue) # Add the SQS subscription to the sns topic MySnsTopic.add_subscription(sqsSubscription) # Add policy statement to SQS Policy that is created as part of the new queue iam.PolicyStatement(actions=['SQS:SendMessage'], effect=iam.Effect.ALLOW, conditions={'ArnEquals': MySnsTopic.topic_arn}, resources=[MySqsQueue.queue_arn], principals=[ iam.ServicePrincipal('sns.amazonaws.com') ] ) CfnOutput(self, "SQS queue name", description="SQS queue name", value=MySqsQueue.queue_name) CfnOutput(self, "SQS queue ARN", description="SQS queue arn", value=MySqsQueue.queue_arn) CfnOutput(self, "SQS queue URL", description="SQS queue URL", value=MySqsQueue.queue_url) CfnOutput(self, "SNS topic name", description="SNS topic name", value=MySnsTopic.topic_name) CfnOutput(self, "SNS topic ARN", description="SNS topic ARN", value=MySnsTopic.topic_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue( self, "CbsappQueue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "CbsappTopic") table = dynamodb.Table(self, id='dynamoTable', table_name='testcdktable', partition_key=dynamodb.Attribute( name='lastname', type=dynamodb.AttributeType.STRING)) function = _lambda.Function(scope=self, id='lambdafunction', function_name='_lambda', code=_lambda.Code.asset('lambdacode'), handler='lambdahandler.main', runtime=_lambda.Runtime.PROVIDED, memory_size=512, timeout=core.Duration.seconds(120)) topic.add_subscription(subs.SqsSubscription(queue))
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) queue = sqs.Queue(self, "sqsqueue", visibility_timeout=core.Duration.seconds(300)) topic = sns.Topic(self, "snstopic") topic.add_subscription(subs.SqsSubscription(queue)) bucket = s3.Bucket( self, "s3Bucket", encryption=s3.BucketEncryption.KMS_MANAGED, block_public_access=s3.BlockPublicAccess.BLOCK_ALL, removal_policy=core.RemovalPolicy.DESTROY, ) bucket.add_event_notification(s3.EventType.OBJECT_CREATED_PUT, s3n.SnsDestination(topic)) s3deploy.BucketDeployment( self, "DeployFile", sources=[s3deploy.Source.asset("./assets")], destination_bucket=bucket, retain_on_delete=False, )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue( self, "PycdkworkshopQueue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "PycdkworkshopTopic") bucket = s3.Bucket(self, id='s3cdkbucket', versioned=True) lambdafunction = _lambda.Function( self, id='lambdafunction', runtime=_lambda.Runtime.PYTHON_3_7, handler='hello.handler', code=_lambda.Code.from_asset(path='lambdacode')) lambdaapi = api.LambdaRestApi(self, id='restapi', handler=lambdafunction) topic.add_subscription(subs.SqsSubscription(queue))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue( self, "HelloQueue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "HelloTopic") topic.add_subscription(subs.SqsSubscription(queue))
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) queue = sqs.Queue( self, "%name.PascalCased%Queue", visibility_timeout=Duration.seconds(300), ) topic = sns.Topic( self, "%name.PascalCased%Topic" ) topic.add_subscription(subs.SqsSubscription(queue))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue(self, "MyFirstQueue", visibility_timeout=core.Duration.seconds(300)) topic = sns.Topic(self, "MyFirstTopic", display_name="My First Topic") topic.add_subscription(subs.SqsSubscription(queue)) hello = CloudendureConstruct(self, "MyHelloConstruct", num_buckets=4) user = iam.User(self, "MyUser") hello.grant_read(user)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) s3.Bucket(self, "myBucketId") queue = sqs.Queue( self, "MyFirstCdkProjectQueue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "MyFirstCdkProjectTopic") topic.add_subscription(subs.SqsSubscription(queue))
def __init__(self, app: cdk.App, id: str, **kwargs) -> None: super().__init__(app, id, **kwargs) queue = sqs.Queue( self, "MyFirstQueue", visibility_timeout_sec=300, ) topic = sns.Topic(self, "MyFirstTopic", display_name="My First Topic") topic.add_subscription(subs.SqsSubscription(queue)) hello = HelloConstruct(self, "MyHelloConstruct", num_buckets=4) user = iam.User(self, "MyUser") hello.grant_read(user)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue( self, "Lab10Queue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "Lab10Topic") topic.add_subscription(subs.SqsSubscription(queue)) vpc = ec2.Vpc.from_lookup(self, "aws-cdk-handson-lab02-vpc", vpc_name="default") # The code that defines your stack goes here core.CfnOutput(self, "CDKMetadata", value="hello cdk")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Define S3 bucket my_bucket = s3.Bucket(self, "ssl-s3-sns-event-raw") #Add Filters if required filter1 = s3.NotificationKeyFilter(prefix="home/") #sns Topic my_sns = sns.Topic(self, id="my-sns-topic", display_name="my-sns-topic") #Create the s3 notification objects which points to Lambda notification = notifications.SnsDestination(my_sns) #link s3 and sns my_bucket.add_event_notification(s3.EventType.OBJECT_CREATED, notification, filter1) #create sqs queue my_sqs = sqs.Queue(self, id="my-queue") #create sqs / sns subcription subscription = aws_sns_subscriptions.SqsSubscription(my_sqs) #add subscription to sns. my_sns.add_subscription(subscription) #create lambda function my_lambda = _lambda.Function(self, "HelloHandler", runtime=_lambda.Runtime.PYTHON_3_7, handler="hello.handler", code=_lambda.Code.asset('lambda')) #create sns/lambda subscription subscription = aws_sns_subscriptions.LambdaSubscription(my_lambda) #add lambda subscription to sns my_sns.add_subscription(subscription)
def __init__(self, scope: core.Construct, id: str, queue_name, **kwargs) -> None: self.name_check = self.check_name_of_queue(queue_name) super().__init__(scope, id, **kwargs) if self.name_check == True: queue = sqs.Queue( self, queue_name, visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "CdkTopic") topic.add_subscription(subs.SqsSubscription(queue)) else: print("Check your queue name pls") exit(1)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) dispatcher = sns.Topic(self, 'TestDispatcher') core.CfnOutput(self, 'testDispatcher', value=dispatcher.topic_arn, export_name='testDispatcherArn') for queueName in ['q1', 'q2', 'q3', 'q4']: q = sqs.Queue(self, queueName) dispatcher.add_subscription( subscriptions.SqsSubscription( q, raw_message_delivery=True, )) core.CfnOutput(self, f'{queueName}Url', value=q.queue_url, export_name=f'{queueName}Url')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue( self, "SlsApiQueue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic( self, "SlsApiTopic" ) topic.add_subscription(subs.SqsSubscription(queue)) ## S1. Define the table that maps short codes to URLs. table = aws_dynamodb.Table(self, "mapping-table", partition_key=aws_dynamodb.Attribute( name="id", type=aws_dynamodb.AttributeType.STRING), read_capacity=10, write_capacity=5) ## S2.1. Defines Lambda resource & API-Gateway request handler ## All API requests will go to the same function. handler = aws_lambda.Function(self, "SlsApiFunction", code=aws_lambda.Code.asset("./lambda"), handler="handler.main", timeout=core.Duration.minutes(5), runtime=aws_lambda.Runtime.PYTHON_3_7) ## S2.2. Pass the table name to the handler through an env variable ## and grant the handler read/write permissions on the table. table.grant_read_write_data(handler) handler.add_environment('TABLE_NAME', table.table_name) ## S3. Define the API endpoint and associate the handler api = aws_apigateway.LambdaRestApi(self, "SlsApiGateway", handler=handler)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) queue = sqs.Queue(self, "StartProwlerScan", receive_message_wait_time=core.Duration.seconds(20), visibility_timeout=core.Duration.seconds(7200)) push_all_active_accounts_onto_queue_lambda_function = lambda_.Function( self, "PushAllActiveAccountsOntoQueue", runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.asset("lambda/pushAllActiveActivesOntoQueue"), handler="lambda_function.lambda_handler", environment={"SQS_QUEUE_URL": queue.queue_url}) event_lambda_target = events_targets.LambdaFunction( handler=push_all_active_accounts_onto_queue_lambda_function) queue.grant_send_messages( push_all_active_accounts_onto_queue_lambda_function) schedule = events.Schedule.rate(core.Duration.days(1)) events.Rule(self, "DailyTrigger", schedule=schedule, targets=[event_lambda_target]) vpc = ec2.Vpc(self, "Vpc") cluster = ecs.Cluster(self, "Cluster", vpc=vpc) logging = ecs.AwsLogDriver(stream_prefix="ProwlerTask", log_retention=logs.RetentionDays.ONE_DAY) results_bucket = s3.Bucket(self, "ResultsBucket") dockerfile_directory = path.join(path.dirname(path.realpath(__file__)), "docker") image = ecr_assets.DockerImageAsset(self, "ProwlerImageBuild", directory=dockerfile_directory) prowler_task = ecs.FargateTaskDefinition(self, "ProwlerTaskDefinition", cpu=256, memory_limit_mib=512) prowler_task.add_container( "Prowler_image", image=ecs.ContainerImage.from_docker_image_asset(image), logging=logging, environment={ "RESULTS_BUCKET": results_bucket.bucket_name, "SQS_QUEUE_URL": queue.queue_url }) task_role = prowler_task.task_role task_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name("ReadOnlyAccess")) queue.grant(task_role, "sqs:DeleteMessage") results_bucket.grant_put(task_role) task_role.attach_inline_policy( iam.Policy(self, "AssumeRolePermissions", statements=[ iam.PolicyStatement(actions=["sts:AssumeRole"], effect=iam.Effect.ALLOW, resources=["*"]) ])) run_fargate_task_lambda_function = lambda_.Function( self, "RunFargateTask", runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.asset("lambda/runFargateTask"), handler="lambda_function.lambda_handler", environment={ "CLUSTER_ARN": cluster.cluster_arn, "SUBNET_IDS": json.dumps( [subnet.subnet_id for subnet in vpc.private_subnets]), "QUEUE_URL": queue.queue_url, "TASK_DEFINITION_ARN": prowler_task.task_definition_arn }) queue.grant(run_fargate_task_lambda_function, "sqs:GetQueueAttributes") sqs_alarm_topic = sns.Topic(self, "SqsAlarmTopic") sqs_alarm_topic.grant_publish(run_fargate_task_lambda_function) sqs_alarm_queue = sqs.Queue( self, "SqsAlarmQueue", retention_period=core.Duration.days(14), visibility_timeout=core.Duration.minutes(3)) sqs_alarm_topic.add_subscription( sns_subscriptions.SqsSubscription(sqs_alarm_queue)) run_fargate_task_lambda_function.add_event_source( lambda_event_sources.SqsEventSource(sqs_alarm_queue)) run_fargate_task_lambda_function.add_to_role_policy( iam.PolicyStatement(actions=["ecs:RunTask"], effect=iam.Effect.ALLOW, resources=[prowler_task.task_definition_arn])) run_fargate_task_lambda_function.add_to_role_policy( iam.PolicyStatement(actions=["iam:PassRole"], effect=iam.Effect.ALLOW, resources=[ prowler_task.execution_role.role_arn, prowler_task.task_role.role_arn ])) sqs_ok_topic = sns.Topic(self, "SqsOkTopic") clear_alarm_queue = lambda_.Function( self, "ClearAlarmQueue", runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.Code.asset("lambda/clearAlarmQueue"), handler="lambda_function.lambda_handler", environment={"QUEUE_URL": sqs_alarm_queue.queue_url}) clear_alarm_queue.add_event_source( lambda_event_sources.SnsEventSource(sqs_ok_topic)) sqs_alarm_queue.grant(clear_alarm_queue, "sqs:DeleteMessage") alarm = cloudwatch.Alarm( self, "FargateTaskTrigger", metric=queue.metric_approximate_number_of_messages_visible( period=core.Duration.seconds(60), statistic="max"), evaluation_periods=1, threshold=1, alarm_description="Run a fargate task when there " "are messages in the queue", treat_missing_data=cloudwatch.TreatMissingData.IGNORE) alarm.add_alarm_action(cloudwatch_actions.SnsAction(sqs_alarm_topic)) alarm.add_ok_action(cloudwatch_actions.SnsAction(sqs_ok_topic))
def __init__( self, scope: core.Construct, id: str, handler: str, memory: int = 3008, timeout: int = 900, concurrent: int = 10, retry: int = 0, permissions: Optional[List[iam.PolicyStatement]] = None, environment: Dict = {}, code_dir: str = "./", **kwargs: Any, ) -> None: """Create AWS Lambda watchbot stack. """ super().__init__(scope, id, **kwargs) permissions = permissions or [] topic = sns.Topic(self, "lambdaTopic", display_name="Lambda Watchbot SNS Topic") core.CfnOutput( self, "SNSTopic", value=topic.topic_arn, description="SNS Topic ARN", export_name="SNSTopic", ) dlqueue = sqs.Queue(self, "lambdaDeadLetterQueue") queue = sqs.Queue( self, "lambdaQueue", visibility_timeout=core.Duration.seconds(timeout), dead_letter_queue=sqs.DeadLetterQueue(queue=dlqueue, max_receive_count=3), ) core.CfnOutput( self, "SQSQueueURL", value=queue.queue_url, description="SQS URL", export_name="SQSQueueURL", ) topic.add_subscription(sns_sub.SqsSubscription(queue)) asset = aws_lambda.AssetImageCode(directory="./") worker = aws_lambda.Function( self, f"{id}-lambda", description="Watchbot's worker", code=asset, handler=aws_lambda.Handler.FROM_IMAGE, runtime=aws_lambda.Runtime.FROM_IMAGE, memory_size=memory, reserved_concurrent_executions=concurrent, timeout=core.Duration.seconds(timeout), retry_attempts=retry, environment=environment, ) for perm in permissions: worker.add_to_role_policy(perm) worker.add_event_source( aws_lambda_event_sources.SqsEventSource(queue, batch_size=1)) topic.grant_publish(worker)
def __init__(self, scope: cdk.Construct, construct_id: str, stack_log_level: str, store_events_topic, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) # Add your stack resources below) # Sales Queue Consumer sales_q = _sqs.Queue( self, "salesEventsQueue", delivery_delay=cdk.Duration.seconds(5), queue_name=f"sales_q", retention_period=cdk.Duration.days(2), visibility_timeout=cdk.Duration.seconds(10), receive_message_wait_time=cdk.Duration.seconds(10)) # Create a Filter for Sales Subscription sales_policy = { "evnt_type": _sns.SubscriptionFilter(conditions=["sales-event"]) } # Create an SQS type subscription to SNS sales_subs = _sns_subs.SqsSubscription(sales_q, filter_policy=sales_policy) # Add the scription to topic store_events_topic.add_subscription(sales_subs) # Read Lambda Code try: with open( "stacks/back_end/serverless_sns_consumer_stack/lambda_src/sqs_data_consumer.py", encoding="utf-8", mode="r") as f: msg_consumer_fn_code = f.read() except OSError: print("Unable to read Lambda Function Code") raise msg_consumer_fn = _lambda.Function( self, "msgConsumerFn", function_name=f"sales_queue_consumer_fn", description="Process messages in SQS queue", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.InlineCode(msg_consumer_fn_code), handler="index.lambda_handler", timeout=cdk.Duration.seconds(5), reserved_concurrent_executions=1, environment={ "LOG_LEVEL": f"{stack_log_level}", "APP_ENV": "Production", "SALES_QUEUE_NAME": f"{sales_q.queue_name}", "TRIGGER_RANDOM_DELAY": "True" }) msg_consumer_fn_version = msg_consumer_fn.latest_version msg_consumer_fn_version_alias = _lambda.Alias( self, "msgConsumerFnAlias", alias_name="MystiqueAutomation", version=msg_consumer_fn_version) # Create Custom Loggroup for Producer msg_consumer_fn_lg = _logs.LogGroup( self, "msgConsumerFnLogGroup", log_group_name=f"/aws/lambda/{msg_consumer_fn.function_name}", removal_policy=cdk.RemovalPolicy.DESTROY, retention=_logs.RetentionDays.ONE_DAY) # Restrict Produce Lambda to be invoked only from the stack owner account msg_consumer_fn.add_permission("restrictLambdaInvocationToOwnAccount", principal=_iam.AccountRootPrincipal(), action="lambda:InvokeFunction", source_account=cdk.Aws.ACCOUNT_ID, source_arn=sales_q.queue_arn) # Set our Lambda Function to be invoked by SQS msg_consumer_fn.add_event_source(_sqsEventSource(sales_q, batch_size=5)) ########################################### ################# OUTPUTS ################# ########################################### output_0 = cdk.CfnOutput( self, "AutomationFrom", value=f"{GlobalArgs.SOURCE_INFO}", description= "To know more about this automation stack, check out our github page." ) output_2 = cdk.CfnOutput( self, "SalesEventsConsumer", value= f"https://console.aws.amazon.com/lambda/home?region={cdk.Aws.REGION}#/functions/{msg_consumer_fn.function_name}", description="Process events received from SQS event bus")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here ########################################################################### # AWS SECRETS MANAGER - Templated secret ########################################################################### # templated_secret = aws_secretsmanager.Secret(self, "TemplatedSecret", # generate_secret_string=aws_secretsmanager.SecretStringGenerator( # secret_string_template= "{\"username\":\"cleanbox\"}", # generate_string_key="password" # ) # ) ########################################################################### # CUSTOM CLOUDFORMATION RESOURCE ########################################################################### # customlambda = aws_lambda.Function(self,'customconfig', # handler='customconfig.on_event', # runtime=aws_lambda.Runtime.PYTHON_3_7, # code=aws_lambda.Code.asset('customconfig'), # ) # customlambda_statement = aws_iam.PolicyStatement(actions=["events:PutRule"], conditions=None, effect=None, not_actions=None, not_principals=None, not_resources=None, principals=None, resources=["*"], sid=None) # customlambda.add_to_role_policy(statement=customlambda_statement) # my_provider = cr.Provider(self, "MyProvider", # on_event_handler=customlambda, # # is_complete_handler=is_complete, # optional async "waiter" # log_retention=logs.RetentionDays.SIX_MONTHS # ) # CustomResource(self, 'customconfigresource', service_token=my_provider.service_token) ########################################################################### # AWS LAMBDA FUNCTIONS ########################################################################### sqs_to_elastic_cloud = aws_lambda.Function( self, 'sqs_to_elastic_cloud', handler='sqs_to_elastic_cloud.lambda_handler', runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset('sqs_to_elastic_cloud'), memory_size=4096, timeout=core.Duration.seconds(300), log_retention=logs.RetentionDays.ONE_DAY) sqs_to_elasticsearch_service = aws_lambda.Function( self, 'sqs_to_elasticsearch_service', handler='sqs_to_elasticsearch_service.lambda_handler', runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset('sqs_to_elasticsearch_service'), memory_size=4096, timeout=core.Duration.seconds(300), log_retention=logs.RetentionDays.ONE_DAY) # sqs_to_elasticsearch_service.add_environment("kinesis_firehose_name", "-") # sqs_to_elastic_cloud.add_environment("index_name", "-") ########################################################################### # AWS LAMBDA FUNCTIONS ########################################################################### # sqs_to_elasticsearch_service_permission = aws_lambda.Permission(*, principal, action=None, event_source_token=None, scope=None, source_account=None, source_arn=None) ########################################################################### # AMAZON S3 BUCKETS ########################################################################### access_log_bucket = aws_s3.Bucket(self, "access_log_bucket") kinesis_log_bucket = aws_s3.Bucket(self, "kinesis_log_bucket") ########################################################################### # LAMBDA SUPPLEMENTAL POLICIES ########################################################################### lambda_supplemental_policy_statement = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=["s3:Get*", "s3:Head*", "s3:List*", "firehose:*"], resources=["*"]) sqs_to_elastic_cloud.add_to_role_policy( lambda_supplemental_policy_statement) sqs_to_elasticsearch_service.add_to_role_policy( lambda_supplemental_policy_statement) ########################################################################### # AWS SNS TOPICS ########################################################################### access_log_topic = aws_sns.Topic(self, "access_log_topic") ########################################################################### # ADD AMAZON S3 BUCKET NOTIFICATIONS ########################################################################### access_log_bucket.add_event_notification( aws_s3.EventType.OBJECT_CREATED, aws_s3_notifications.SnsDestination(access_log_topic)) ########################################################################### # AWS SQS QUEUES ########################################################################### sqs_to_elasticsearch_service_queue_iqueue = aws_sqs.Queue( self, "sqs_to_elasticsearch_service_queue_dlq") sqs_to_elasticsearch_service_queue_dlq = aws_sqs.DeadLetterQueue( max_receive_count=10, queue=sqs_to_elasticsearch_service_queue_iqueue) sqs_to_elasticsearch_service_queue = aws_sqs.Queue( self, "sqs_to_elasticsearch_service_queue", visibility_timeout=core.Duration.seconds(301), dead_letter_queue=sqs_to_elasticsearch_service_queue_dlq) sqs_to_elastic_cloud_queue_iqueue = aws_sqs.Queue( self, "sqs_to_elastic_cloud_queue_dlq") sqs_to_elastic_cloud_queue_dlq = aws_sqs.DeadLetterQueue( max_receive_count=10, queue=sqs_to_elastic_cloud_queue_iqueue) sqs_to_elastic_cloud_queue = aws_sqs.Queue( self, "sqs_to_elastic_cloud_queue", visibility_timeout=core.Duration.seconds(301), dead_letter_queue=sqs_to_elastic_cloud_queue_dlq) ########################################################################### # AWS SNS TOPIC SUBSCRIPTIONS ########################################################################### access_log_topic.add_subscription( aws_sns_subscriptions.SqsSubscription(sqs_to_elastic_cloud_queue)) access_log_topic.add_subscription( aws_sns_subscriptions.SqsSubscription( sqs_to_elasticsearch_service_queue)) ########################################################################### # AWS LAMBDA SQS EVENT SOURCE ########################################################################### sqs_to_elastic_cloud.add_event_source( SqsEventSource(sqs_to_elastic_cloud_queue, batch_size=10)) sqs_to_elasticsearch_service.add_event_source( SqsEventSource(sqs_to_elasticsearch_service_queue, batch_size=10)) ########################################################################### # AWS ELASTICSEARCH DOMAIN ########################################################################### ########################################################################### # AWS ELASTICSEARCH DOMAIN ACCESS POLICY ########################################################################### this_aws_account = aws_iam.AccountPrincipal(account_id="012345678912") # s3_to_elasticsearch_access_logs_domain_access_policy_statement = aws_iam.PolicyStatement( # principals=[this_aws_account], # effect=aws_iam.Effect.ALLOW, # actions=["es:*"], # resources=["*"] # ) # s3_to_elasticsearch_access_logs_domain_access_policy_statement_list=[] # s3_to_elasticsearch_access_logs_domain_access_policy_statement_list.append(s3_to_elasticsearch_access_logs_domain_access_policy_statement) s3_to_elasticsearch_access_logs_domain = aws_elasticsearch.Domain( self, "s3-to-elasticsearch-access-logs-domain", # access_policies=s3_to_elasticsearch_access_logs_domain_access_policy_statement_list, version=aws_elasticsearch.ElasticsearchVersion.V7_1, capacity={ "master_nodes": 3, "data_nodes": 4 }, ebs={"volume_size": 100}, zone_awareness={"availability_zone_count": 2}, logging={ "slow_search_log_enabled": True, "app_log_enabled": True, "slow_index_log_enabled": True }) ########################################################################### # AMAZON COGNITO USER POOL ########################################################################### s3_to_elasticsearch_user_pool = aws_cognito.UserPool( self, "s3-to-elasticsearch-access-logs-pool", account_recovery=None, auto_verify=None, custom_attributes=None, email_settings=None, enable_sms_role=None, lambda_triggers=None, mfa=None, mfa_second_factor=None, password_policy=None, self_sign_up_enabled=None, sign_in_aliases=aws_cognito.SignInAliases(email=True, phone=None, preferred_username=None, username=True), sign_in_case_sensitive=None, sms_role=None, sms_role_external_id=None, standard_attributes=None, user_invitation=None, user_pool_name=None, user_verification=None) ########################################################################### # AMAZON KINESIS FIREHOSE STREAM ########################################################################### # kinesis_policy_statement = aws_iam.PolicyStatement( # effect=aws_iam.Effect.ALLOW, # # actions=["es:*", "s3:*", "kms:*", "kinesis:*", "lambda:*"], # actions=["*"], # resources=["*"] # ) # kinesis_policy_document = aws_iam.PolicyDocument() # kinesis_policy_document.add_statements(kinesis_policy_statement) kinesis_firehose_stream_role = aws_iam.Role( self, "BaseVPCIAMLogRole", assumed_by=aws_iam.ServicePrincipal('firehose.amazonaws.com'), role_name=None, inline_policies={ "AllowLogAccess": aws_iam.PolicyDocument( assign_sids=False, statements=[ aws_iam.PolicyStatement(actions=[ '*', 'es:*', 'logs:PutLogEvents', 'logs:DescribeLogGroups', 'logs:DescribeLogsStreams' ], effect=aws_iam.Effect('ALLOW'), resources=['*']) ]) }) RetryOptions = aws_kinesisfirehose.CfnDeliveryStream.ElasticsearchRetryOptionsProperty( duration_in_seconds=300) s3_configuration = aws_kinesisfirehose.CfnDeliveryStream.S3DestinationConfigurationProperty( bucket_arn=kinesis_log_bucket.bucket_arn, role_arn=kinesis_firehose_stream_role.role_arn) ElasticsearchDestinationConfiguration = aws_kinesisfirehose.CfnDeliveryStream.ElasticsearchDestinationConfigurationProperty( # "BufferingHints" : ElasticsearchBufferingHints, # "CloudWatchLoggingOptions" : CloudWatchLoggingOptions, # "ClusterEndpoint" : String, domain_arn=s3_to_elasticsearch_access_logs_domain.domain_arn, index_name="s3-to-elasticsearch-accesslogs", index_rotation_period="OneDay", # "ProcessingConfiguration" : ProcessingConfiguration, retry_options=RetryOptions, role_arn=kinesis_firehose_stream_role.role_arn, # "S3BackupMode" : String, s3_configuration=s3_configuration # "TypeName" : String # "VpcConfiguration" : VpcConfiguration ) kinesis_firehose_stream = aws_kinesisfirehose.CfnDeliveryStream( self, "kinesis_firehose_stream", delivery_stream_encryption_configuration_input=None, delivery_stream_name=None, delivery_stream_type=None, elasticsearch_destination_configuration= ElasticsearchDestinationConfiguration, extended_s3_destination_configuration=None, http_endpoint_destination_configuration=None, kinesis_stream_source_configuration=None, redshift_destination_configuration=None, s3_destination_configuration=None, splunk_destination_configuration=None, tags=None) sqs_to_elasticsearch_service.add_environment( "FIREHOSE_NAME", kinesis_firehose_stream.ref) sqs_to_elasticsearch_service.add_environment( "QUEUEURL", sqs_to_elasticsearch_service_queue.queue_url) sqs_to_elasticsearch_service.add_environment("DEBUG", "False") sqs_to_elastic_cloud.add_environment("ELASTICCLOUD_SECRET_NAME", "-") sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_ID", "-") sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_PASSWORD", "-") sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_USERNAME", "-") sqs_to_elastic_cloud.add_environment( "QUEUEURL", sqs_to_elastic_cloud_queue.queue_url) sqs_to_elastic_cloud.add_environment("DEBUG", "False")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here ########################################################################### # AWS SECRETS MANAGER - Templated secret ########################################################################### # templated_secret = aws_secretsmanager.Secret(self, "TemplatedSecret", # generate_secret_string=aws_secretsmanager.SecretStringGenerator( # secret_string_template= "{\"username\":\"cleanbox\"}", # generate_string_key="password" # ) # ) ########################################################################### # CUSTOM CLOUDFORMATION RESOURCE ########################################################################### # customlambda = aws_lambda.Function(self,'customconfig', # handler='customconfig.on_event', # runtime=aws_lambda.Runtime.PYTHON_3_7, # code=aws_lambda.Code.asset('customconfig'), # ) # customlambda_statement = aws_iam.PolicyStatement(actions=["events:PutRule"], conditions=None, effect=None, not_actions=None, not_principals=None, not_resources=None, principals=None, resources=["*"], sid=None) # customlambda.add_to_role_policy(statement=customlambda_statement) # my_provider = cr.Provider(self, "MyProvider", # on_event_handler=customlambda, # # is_complete_handler=is_complete, # optional async "waiter" # log_retention=logs.RetentionDays.SIX_MONTHS # ) # CustomResource(self, 'customconfigresource', service_token=my_provider.service_token) ########################################################################### # AWS LAMBDA FUNCTIONS ########################################################################### sqs_to_elastic_cloud = aws_lambda.Function( self, 'sqs_to_elastic_cloud', handler='sqs_to_elastic_cloud.lambda_handler', runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset('sqs_to_elastic_cloud'), memory_size=4096, timeout=core.Duration.seconds(301), log_retention=logs.RetentionDays.ONE_DAY) sqs_to_elasticsearch_service = aws_lambda.Function( self, 'sqs_to_elasticsearch_service', handler='sqs_to_elasticsearch_service.lambda_handler', runtime=aws_lambda.Runtime.PYTHON_3_7, code=aws_lambda.Code.asset('sqs_to_elasticsearch_service'), memory_size=4096, timeout=core.Duration.seconds(301), log_retention=logs.RetentionDays.ONE_DAY) ########################################################################### # AWS LAMBDA FUNCTIONS ########################################################################### ########################################################################### # AMAZON S3 BUCKETS ########################################################################### cloudtrail_log_bucket = aws_s3.Bucket(self, "cloudtrail_log_bucket") ########################################################################### # LAMBDA SUPPLEMENTAL POLICIES ########################################################################### lambda_supplemental_policy_statement = aws_iam.PolicyStatement( effect=aws_iam.Effect.ALLOW, actions=["s3:Get*", "s3:Head*", "s3:List*", "firehose:*", "es:*"], resources=["*"]) sqs_to_elastic_cloud.add_to_role_policy( lambda_supplemental_policy_statement) sqs_to_elasticsearch_service.add_to_role_policy( lambda_supplemental_policy_statement) ########################################################################### # AWS SNS TOPICS ########################################################################### cloudtrail_log_topic = aws_sns.Topic(self, "cloudtrail_log_topic") ########################################################################### # ADD AMAZON S3 BUCKET NOTIFICATIONS ########################################################################### cloudtrail_log_bucket.add_event_notification( aws_s3.EventType.OBJECT_CREATED, aws_s3_notifications.SnsDestination(cloudtrail_log_topic)) ########################################################################### # AWS SQS QUEUES ########################################################################### sqs_to_elasticsearch_service_queue_iqueue = aws_sqs.Queue( self, "sqs_to_elasticsearch_service_queue_dlq") sqs_to_elasticsearch_service_queue_dlq = aws_sqs.DeadLetterQueue( max_receive_count=10, queue=sqs_to_elasticsearch_service_queue_iqueue) sqs_to_elasticsearch_service_queue = aws_sqs.Queue( self, "sqs_to_elasticsearch_service_queue", visibility_timeout=core.Duration.seconds(300), dead_letter_queue=sqs_to_elasticsearch_service_queue_dlq) sqs_to_elastic_cloud_queue_iqueue = aws_sqs.Queue( self, "sqs_to_elastic_cloud_queue_dlq") sqs_to_elastic_cloud_queue_dlq = aws_sqs.DeadLetterQueue( max_receive_count=10, queue=sqs_to_elastic_cloud_queue_iqueue) sqs_to_elastic_cloud_queue = aws_sqs.Queue( self, "sqs_to_elastic_cloud_queue", visibility_timeout=core.Duration.seconds(300), dead_letter_queue=sqs_to_elastic_cloud_queue_dlq) ########################################################################### # AWS SNS TOPIC SUBSCRIPTIONS ########################################################################### cloudtrail_log_topic.add_subscription( aws_sns_subscriptions.SqsSubscription(sqs_to_elastic_cloud_queue)) cloudtrail_log_topic.add_subscription( aws_sns_subscriptions.SqsSubscription( sqs_to_elasticsearch_service_queue)) ########################################################################### # AWS LAMBDA SQS EVENT SOURCE ########################################################################### sqs_to_elastic_cloud.add_event_source( SqsEventSource(sqs_to_elastic_cloud_queue, batch_size=10)) sqs_to_elasticsearch_service.add_event_source( SqsEventSource(sqs_to_elasticsearch_service_queue, batch_size=10)) ########################################################################### # AWS ELASTICSEARCH DOMAIN ########################################################################### ########################################################################### # AWS ELASTICSEARCH DOMAIN ACCESS POLICY ########################################################################### this_aws_account = aws_iam.AccountPrincipal(account_id="012345678912") s3_to_elasticsearch_cloudtrail_logs_domain = aws_elasticsearch.Domain( self, "s3-to-elasticsearch-cloudtrail-logs-domain", version=aws_elasticsearch.ElasticsearchVersion.V7_1, capacity={ "master_nodes": 3, "data_nodes": 4 }, ebs={"volume_size": 100}, zone_awareness={"availability_zone_count": 2}, logging={ "slow_search_log_enabled": True, "app_log_enabled": True, "slow_index_log_enabled": True }) ########################################################################### # AMAZON COGNITO USER POOL ########################################################################### s3_to_elasticsearch_user_pool = aws_cognito.UserPool( self, "s3-to-elasticsearch-cloudtrial-logs-pool", account_recovery=None, auto_verify=None, custom_attributes=None, email_settings=None, enable_sms_role=None, lambda_triggers=None, mfa=None, mfa_second_factor=None, password_policy=None, self_sign_up_enabled=None, sign_in_aliases=aws_cognito.SignInAliases(email=True, phone=None, preferred_username=None, username=True), sign_in_case_sensitive=None, sms_role=None, sms_role_external_id=None, standard_attributes=None, user_invitation=None, user_pool_name=None, user_verification=None) sqs_to_elasticsearch_service.add_environment( "ELASTICSEARCH_HOST", s3_to_elasticsearch_cloudtrail_logs_domain.domain_endpoint) sqs_to_elasticsearch_service.add_environment( "QUEUEURL", sqs_to_elasticsearch_service_queue.queue_url) sqs_to_elasticsearch_service.add_environment("DEBUG", "False") sqs_to_elastic_cloud.add_environment("ELASTICCLOUD_SECRET_NAME", "-") sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_ID", "-") sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_PASSWORD", "-") sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_USERNAME", "-") sqs_to_elastic_cloud.add_environment( "QUEUEURL", sqs_to_elastic_cloud_queue.queue_url) sqs_to_elastic_cloud.add_environment("DEBUG", "False") ########################################################################### # AWS COGNITO USER POOL ########################################################################### allevents_trail = aws_cloudtrail.Trail( self, "allevents_trail", bucket=cloudtrail_log_bucket, cloud_watch_log_group=None, cloud_watch_logs_retention=None, enable_file_validation=None, encryption_key=None, include_global_service_events=None, is_multi_region_trail=True, kms_key=None, management_events=aws_cloudtrail.ReadWriteType("ALL"), s3_key_prefix=None, send_to_cloud_watch_logs=False, sns_topic=None, trail_name=None)
def __init__(self, scope: core.Construct, id: str, infra: RtspBaseResourcesConstruct, **kwargs) -> None: super().__init__(scope, id, **kwargs) # Create the inventory bucket... self.inventories = s3.Bucket( self, 'InventoryBucket', bucket_name='homenet-{}.rtsp-inventories.{}.virtual.world'.format( infra.landing_zone.zone_name, core.Stack.of(self).region).lower(), removal_policy=core.RemovalPolicy.DESTROY, cors=[ s3.CorsRule(allowed_methods=[s3.HttpMethods.GET], allowed_origins=['*']) ], lifecycle_rules=[ s3.LifecycleRule( id='Transition-to-IA-after-30D', prefix='eufy/', abort_incomplete_multipart_upload_after=core.Duration.days( 7), transitions=[ s3.Transition( storage_class=s3.StorageClass.INFREQUENT_ACCESS, transition_after=core.Duration.days(30)) ]) ]) # Create inventory collections for the Eufy Homebases... infra.bucket.add_inventory( objects_prefix='eufy/', inventory_id='{}-InventoryReport'.format('EufyFull'), format=s3.InventoryFormat.CSV, frequency=s3.InventoryFrequency.DAILY, include_object_versions=s3.InventoryObjectVersion.CURRENT, destination=s3.InventoryDestination( bucket=self.inventories, bucket_owner=core.Aws.ACCOUNT_ID, prefix=None)) for base_name in ['Moonbase', 'Starbase']: prefix = 'eufy/{}.cameras.real.world/'.format(base_name).lower() infra.bucket.add_inventory( objects_prefix=prefix, inventory_id='{}-InventoryReport'.format(base_name), format=s3.InventoryFormat.CSV, frequency=s3.InventoryFrequency.DAILY, include_object_versions=s3.InventoryObjectVersion.CURRENT, destination=s3.InventoryDestination( bucket=self.inventories, bucket_owner=core.Aws.ACCOUNT_ID, prefix=None)) # Broadcast inventory creation events... self.inventoryAvailable = sns.Topic( self, 'InventoryAvailable', display_name='HomeNet-{}-Rtsp-InventoryAvailable'.format( infra.landing_zone.zone_name), topic_name='HomeNet-{}-Rtsp-InventoryAvailable'.format( infra.landing_zone.zone_name)) self.inventories.add_event_notification( s3.EventType.OBJECT_CREATED, s3n.SnsDestination(topic=self.inventoryAvailable), s3.NotificationKeyFilter(suffix='manifest.json')) # Attach debug queue to the notification self.inventoryAvailable.add_subscription( subs.SqsSubscription( raw_message_delivery=True, queue=sqs.Queue( self, 'InventoryDebugQueue', removal_policy=core.RemovalPolicy.DESTROY, retention_period=core.Duration.days(7), queue_name='HomeNet-{}-RtspInventoryAvailable_Debug'. format(infra.landing_zone.zone_name)))) # Subscribe the GroundTruth Manifest Generator groundtruth = RtspGroundTruthManifestGenerationFunction( self, 'GroundTruthManifest', infra=infra, topic=self.inventoryAvailable) self.inventories.grant_read_write(groundtruth.function.role) # Create the RtspNormalizeImage S3 Object Lambda RtspNormalizeImageAccessPoint(scope=self, id='NormalizedImage', infra=infra)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id) lambda_dir = kwargs["lambda_dir"] # Note: A dead-letter queue is optional but it helps capture any failed messages dlq = sqs.Queue(self, id="dead_letter_queue_id", retention_period=Duration.days(7)) dead_letter_queue = sqs.DeadLetterQueue(max_receive_count=1, queue=dlq) upload_queue = sqs.Queue(self, id="sample_queue_id", visibility_timeout=Duration.seconds(30), dead_letter_queue=dead_letter_queue) sqs_subscription = sns_subs.SqsSubscription(upload_queue, raw_message_delivery=True) upload_event_topic = sns.Topic(self, id="sample_sns_topic_id") # This binds the SNS Topic to the SQS Queue upload_event_topic.add_subscription(sqs_subscription) # Note: Lifecycle Rules are optional but are included here to keep costs # low by cleaning up old files or moving them to lower cost storage options s3_bucket = s3.Bucket( self, id="sample_bucket_id", block_public_access=s3.BlockPublicAccess.BLOCK_ALL, versioned=True, lifecycle_rules=[ s3.LifecycleRule( enabled=True, expiration=Duration.days(365), transitions=[ s3.Transition( storage_class=s3.StorageClass.INFREQUENT_ACCESS, transition_after=Duration.days(30)), s3.Transition(storage_class=s3.StorageClass.GLACIER, transition_after=Duration.days(90)), ]) ]) # Note: If you don't specify a filter all uploads will trigger an event. # Also, modifying the event type will handle other object operations # This binds the S3 bucket to the SNS Topic s3_bucket.add_event_notification( s3.EventType.OBJECT_CREATED_PUT, s3n.SnsDestination(upload_event_topic), s3.NotificationKeyFilter(prefix="uploads", suffix=".csv")) function = _lambda.Function( self, "lambda_function", runtime=_lambda.Runtime.PYTHON_3_9, handler="lambda_function.handler", code=_lambda.Code.from_asset(path=lambda_dir)) # This binds the lambda to the SQS Queue invoke_event_source = lambda_events.SqsEventSource(upload_queue) function.add_event_source(invoke_event_source) # Examples of CloudFormation outputs CfnOutput( self, "UploadFileToS3Example", value="aws s3 cp <local-path-to-file> s3://{}/".format( s3_bucket.bucket_name), description= "Upload a file to S3 (using AWS CLI) to trigger the SQS chain", ) CfnOutput( self, "UploadSqsQueueUrl", value=upload_queue.queue_url, description="Link to the SQS Queue triggered on S3 uploads", ) CfnOutput( self, "LambdaFunctionName", value=function.function_name, ) CfnOutput( self, "LambdaFunctionLogGroupName", value=function.log_group.log_group_name, )
def __init__( self, scope: core.Construct, id: str, entrypoint: Optional[List] = None, cpu: Union[int, float] = 256, memory: Union[int, float] = 512, mincount: int = 0, maxcount: int = 50, scaling_steps: int = 5, permissions: Optional[List[iam.PolicyStatement]] = None, vpc_id: Optional[str] = None, vpc_is_default: Optional[bool] = None, environment: dict = {}, **kwargs: Any, ) -> None: """Define stack.""" super().__init__(scope, id, **kwargs) permissions = permissions or [] vpc = ec2.Vpc.from_lookup(self, "vpc", vpc_id=vpc_id, is_default=vpc_is_default) cluster = ecs.Cluster(self, f"{id}-cluster", vpc=vpc) topic = sns.Topic(self, "ecsTopic", display_name="ECS Watchbot SNS Topic") core.CfnOutput( self, "SNSTopic", value=topic.topic_arn, description="SNS Topic ARN", export_name=f"{id}-SNSTopic", ) dlqueue = sqs.Queue(self, "ecsDeadLetterQueue") core.CfnOutput( self, "DeadSQSQueueURL", value=dlqueue.queue_url, description="DeadLetter SQS URL", export_name=f"{id}-DeadSQSQueueURL", ) queue = sqs.Queue( self, "ecsQueue", dead_letter_queue=sqs.DeadLetterQueue(queue=dlqueue, max_receive_count=3), ) core.CfnOutput( self, "SQSQueueURL", value=queue.queue_url, description="SQS URL", export_name=f"{id}-SQSQueueURL", ) environment.update({ "REGION": self.region, "QUEUE_NAME": queue.queue_name }) topic.add_subscription(sns_sub.SqsSubscription(queue)) fargate_task_definition = ecs.FargateTaskDefinition( self, "FargateTaskDefinition", memory_limit_mib=memory, cpu=cpu, ) log_driver = ecs.AwsLogDriver( stream_prefix=f"/ecs/tilebot/{id}", log_retention=aws_logs.RetentionDays.ONE_WEEK, ) fargate_task_definition.add_container( "FargateContainer", image=ecs.ContainerImage.from_asset(directory="./"), entry_point=entrypoint, environment=environment, logging=log_driver, ) fargate_service = ecs.FargateService( self, "FargateService", cluster=cluster, task_definition=fargate_task_definition, desired_count=mincount, enable_ecs_managed_tags=True, assign_public_ip=True, ) permissions.append( iam.PolicyStatement( actions=["sqs:*"], resources=[queue.queue_arn], )) for perm in permissions: fargate_service.task_definition.task_role.add_to_policy(perm) total_number_of_message_lambda = aws_lambda.Function( self, f"{id}-TotalMessagesLambda", description="Create TotalNumberOfMessage metrics", code=aws_lambda.Code.from_inline("""const AWS = require('aws-sdk'); exports.handler = function(event, context, callback) { const sqs = new AWS.SQS({ region: process.env.AWS_DEFAULT_REGION }); const cw = new AWS.CloudWatch({ region: process.env.AWS_DEFAULT_REGION }); return sqs.getQueueAttributes({ QueueUrl: process.env.SQS_QUEUE_URL, AttributeNames: ['ApproximateNumberOfMessagesNotVisible', 'ApproximateNumberOfMessages'] }).promise() .then((attrs) => { return cw.putMetricData({ Namespace: 'AWS/SQS', MetricData: [{ MetricName: 'TotalNumberOfMessages', Dimensions: [{ Name: 'QueueName', Value: process.env.SQS_QUEUE_NAME }], Value: Number(attrs.Attributes.ApproximateNumberOfMessagesNotVisible) + Number(attrs.Attributes.ApproximateNumberOfMessages) }] }).promise(); }) .then((metric) => callback(null, metric)) .catch((err) => callback(err)); };"""), handler="index.handler", runtime=aws_lambda.Runtime.NODEJS_10_X, timeout=core.Duration.seconds(60), environment={ "SQS_QUEUE_URL": queue.queue_url, "SQS_QUEUE_NAME": queue.queue_name, }, ) total_number_of_message_lambda.add_to_role_policy( iam.PolicyStatement( actions=["sqs:GetQueueAttributes"], resources=[queue.queue_arn], )) total_number_of_message_lambda.add_to_role_policy( iam.PolicyStatement( actions=["cloudwatch:PutMetricData"], resources=["*"], )) total_number_of_message_lambda.add_to_role_policy( iam.PolicyStatement( actions=["logs:*"], resources=["arn:aws:logs:*:*:*"], )) rule = aws_events.Rule( self, "TotalMessagesSchedule", schedule=aws_events.Schedule.rate(core.Duration.seconds(60)), ) rule.add_target( aws_events_targets.LambdaFunction(total_number_of_message_lambda)) scalable_target = auto_scale.ScalableTarget( self, "AutoScallingTarget", min_capacity=mincount, max_capacity=maxcount, service_namespace=auto_scale.ServiceNamespace.ECS, resource_id="/".join([ "service", cluster.cluster_name, fargate_service.service_name ]), scalable_dimension="ecs:service:DesiredCount", ) scalable_target.node.add_dependency(fargate_service) scale_up = auto_scale.CfnScalingPolicy( self, "ScaleUp", policy_name="PolicyScaleUp", policy_type="StepScaling", scaling_target_id=scalable_target.scalable_target_id, step_scaling_policy_configuration=auto_scale.CfnScalingPolicy. StepScalingPolicyConfigurationProperty( adjustment_type="ChangeInCapacity", cooldown=120, metric_aggregation_type="Maximum", step_adjustments=[ auto_scale.CfnScalingPolicy.StepAdjustmentProperty( scaling_adjustment=scaling_steps, metric_interval_lower_bound=0, ), ], ), ) scale_up_trigger = aws_cloudwatch.CfnAlarm( # noqa self, "ScaleUpTrigger", alarm_description="Scale up due to visible messages in queue", dimensions=[ aws_cloudwatch.CfnAlarm.DimensionProperty( name="QueueName", value=queue.queue_name, ), ], metric_name="ApproximateNumberOfMessagesVisible", namespace="AWS/SQS", evaluation_periods=1, comparison_operator="GreaterThanThreshold", period=60, statistic="Maximum", threshold=0, alarm_actions=[scale_up.ref], ) scale_down = auto_scale.CfnScalingPolicy( self, "ScaleDown", policy_name="PolicyScaleDown", policy_type="StepScaling", scaling_target_id=scalable_target.scalable_target_id, step_scaling_policy_configuration=auto_scale.CfnScalingPolicy. StepScalingPolicyConfigurationProperty( adjustment_type="ExactCapacity", cooldown=60, step_adjustments=[ auto_scale.CfnScalingPolicy.StepAdjustmentProperty( scaling_adjustment=mincount, metric_interval_upper_bound=0, ), ], ), ) scale_down_trigger = aws_cloudwatch.CfnAlarm( # noqa self, "ScaleDownTrigger", alarm_description= "Scale down due to lack of in-flight messages in queue", dimensions=[ aws_cloudwatch.CfnAlarm.DimensionProperty( name="QueueName", value=queue.queue_name, ), ], metric_name="TotalNumberOfMessages", namespace="AWS/SQS", evaluation_periods=1, comparison_operator="LessThanThreshold", period=120, statistic="Maximum", threshold=1, alarm_actions=[scale_down.ref], )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here # ********* SNS Topics ************* jobCompletionTopic = sns.Topic(self, "JobCompletion") # **********IAM Roles****************************** textractServiceRole = iam.Role( self, "TextractServiceRole", assumed_by=iam.ServicePrincipal("textract.amazonaws.com"), ) textractServiceRole.add_to_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[jobCompletionTopic.topic_arn], actions=["sns:Publish"], )) comprehendServiceRole = iam.Role( self, "ComprehendServiceRole", assumed_by=iam.ServicePrincipal("comprehend.amazonaws.com"), ) comprehendServiceRole.add_to_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=[ "comprehend:*", "s3:ListAllMyBuckets", "s3:ListBucket", "s3:GetBucketLocation", "iam:ListRoles", "iam:GetRole", ], )) # **********S3 Batch Operations Role****************************** s3BatchOperationsRole = iam.Role( self, "S3BatchOperationsRole", assumed_by=iam.ServicePrincipal( "batchoperations.s3.amazonaws.com"), ) # **********S3 Bucket****************************** # S3 bucket for input documents and output contentBucket = s3.Bucket(self, "DocumentsBucket", versioned=False) existingContentBucket = s3.Bucket(self, "ExistingDocumentsBucket", versioned=False) existingContentBucket.grant_read_write(s3BatchOperationsRole) inventoryAndLogsBucket = s3.Bucket(self, "InventoryAndLogsBucket", versioned=False) inventoryAndLogsBucket.grant_read_write(s3BatchOperationsRole) # **********DynamoDB Table************************* # DynamoDB table with links to output in S3 outputTable = dynamodb.Table( self, "OutputTable", partition_key={ "name": "documentId", "type": dynamodb.AttributeType.STRING, }, sort_key={ "name": "outputType", "type": dynamodb.AttributeType.STRING, }, ) # DynamoDB table with links to output in S3 documentsTable = dynamodb.Table( self, "DocumentsTable", partition_key={ "name": "documentId", "type": dynamodb.AttributeType.STRING, }, stream=dynamodb.StreamViewType.NEW_IMAGE, ) # **********SQS Queues***************************** # DLQ (Dead Letter Queue) dlq = sqs.Queue( self, "DLQ", visibility_timeout=core.Duration.seconds(30), retention_period=core.Duration.seconds(1209600), ) # Input Queue for sync jobs syncJobsQueue = sqs.Queue( self, "SyncJobs", visibility_timeout=core.Duration.seconds(30), retention_period=core.Duration.seconds(1209600), dead_letter_queue={ "queue": dlq, "max_receive_count": 50 }, ) # Input Queue for async jobs asyncJobsQueue = sqs.Queue( self, "AsyncJobs", visibility_timeout=core.Duration.seconds(30), retention_period=core.Duration.seconds(1209600), dead_letter_queue={ "queue": dlq, "max_receive_count": 50 }, ) # Queue jobResultsQueue = sqs.Queue( self, "JobResults", visibility_timeout=core.Duration.seconds(900), retention_period=core.Duration.seconds(1209600), dead_letter_queue={ "queue": dlq, "max_receive_count": 50 }, ) # Trigger # jobCompletionTopic.subscribeQueue(jobResultsQueue) jobCompletionTopic.add_subscription( snsSubscriptions.SqsSubscription(jobResultsQueue)) # **********Lambda Functions****************************** # Helper Layer with helper functions helperLayer = _lambda.LayerVersion( self, "HelperLayer", code=_lambda.Code.from_asset("awscdk/lambda/helper"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], license="Apache-2.0", description="Helper layer.", ) # Textractor helper layer textractorLayer = _lambda.LayerVersion( self, "Textractor", code=_lambda.Code.from_asset("awscdk/lambda/textractor"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], license="Apache-2.0", description="Textractor layer.", ) # ----------------------------------------------------------- # S3 Event processor s3Processor = _lambda.Function( self, "S3Processor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/s3processor"), handler="lambda_function.lambda_handler", environment={ "SYNC_QUEUE_URL": syncJobsQueue.queue_url, "ASYNC_QUEUE_URL": asyncJobsQueue.queue_url, "DOCUMENTS_TABLE": documentsTable.table_name, "OUTPUT_TABLE": outputTable.table_name, }, ) # Layer s3Processor.add_layers(helperLayer) # Trigger s3Processor.add_event_source( S3EventSource(contentBucket, events=[s3.EventType.OBJECT_CREATED])) # Permissions documentsTable.grant_read_write_data(s3Processor) syncJobsQueue.grant_send_messages(s3Processor) asyncJobsQueue.grant_send_messages(s3Processor) # ------------------------------------------------------------ # S3 Batch Operations Event processor s3BatchProcessor = _lambda.Function( self, "S3BatchProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/s3batchprocessor"), handler="lambda_function.lambda_handler", environment={ "DOCUMENTS_TABLE": documentsTable.table_name, "OUTPUT_TABLE": outputTable.table_name, }, reserved_concurrent_executions=1, ) # Layer s3BatchProcessor.add_layers(helperLayer) # Permissions documentsTable.grant_read_write_data(s3BatchProcessor) s3BatchProcessor.grant_invoke(s3BatchOperationsRole) s3BatchOperationsRole.add_to_policy( iam.PolicyStatement(actions=["lambda:*"], resources=["*"])) # ------------------------------------------------------------ # Document processor (Router to Sync/Async Pipeline) documentProcessor = _lambda.Function( self, "TaskProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/documentprocessor"), handler="lambda_function.lambda_handler", environment={ "SYNC_QUEUE_URL": syncJobsQueue.queue_url, "ASYNC_QUEUE_URL": asyncJobsQueue.queue_url, }, ) # Layer documentProcessor.add_layers(helperLayer) # Trigger documentProcessor.add_event_source( DynamoEventSource( documentsTable, starting_position=_lambda.StartingPosition.TRIM_HORIZON, )) # Permissions documentsTable.grant_read_write_data(documentProcessor) syncJobsQueue.grant_send_messages(documentProcessor) asyncJobsQueue.grant_send_messages(documentProcessor) # ------------------------------------------------------------ # Sync Jobs Processor (Process jobs using sync APIs) syncProcessor = _lambda.Function( self, "SyncProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/documentprocessor"), handler="lambda_function.lambda_handler", environment={ "OUTPUT_TABLE": outputTable.table_name, "DOCUMENTS_TABLE": documentsTable.table_name, "AWS_DATA_PATH": "models", }, reserved_concurrent_executions=1, timeout=core.Duration.seconds(25), ) # Layer syncProcessor.add_layers(helperLayer) syncProcessor.add_layers(textractorLayer) # Trigger syncProcessor.add_event_source( SqsEventSource(syncJobsQueue, batch_size=1)) # Permissions contentBucket.grant_read_write(syncProcessor) existingContentBucket.grant_read_write(syncProcessor) outputTable.grant_read_write_data(syncProcessor) documentsTable.grant_read_write_data(syncProcessor) syncProcessor.add_to_role_policy( iam.PolicyStatement(actions=["textract:*"], resources=["*"])) # ------------------------------------------------------------ # Async Job Processor (Start jobs using Async APIs) asyncProcessor = _lambda.Function( self, "ASyncProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/asyncprocessor"), handler="lambda_function.lambda_handler", environment={ "ASYNC_QUEUE_URL": asyncJobsQueue.queue_url, "SNS_TOPIC_ARN": jobCompletionTopic.topic_arn, "SNS_ROLE_ARN": textractServiceRole.role_arn, "AWS_DATA_PATH": "models", }, reserved_concurrent_executions=1, timeout=core.Duration.seconds(60), ) # asyncProcessor.addEnvironment("SNS_TOPIC_ARN", textractServiceRole.topic_arn) # Layer asyncProcessor.add_layers(helperLayer) # Triggers # Run async job processor every 5 minutes # Enable code below after test deploy rule = events.Rule( self, "Rule", schedule=events.Schedule.expression("rate(2 minutes)")) rule.add_target(LambdaFunction(asyncProcessor)) # Run when a job is successfully complete asyncProcessor.add_event_source(SnsEventSource(jobCompletionTopic)) # Permissions contentBucket.grant_read(asyncProcessor) existingContentBucket.grant_read_write(asyncProcessor) asyncJobsQueue.grant_consume_messages(asyncProcessor) asyncProcessor.add_to_role_policy( iam.PolicyStatement( actions=["iam:PassRole"], resources=[textractServiceRole.role_arn], )) asyncProcessor.add_to_role_policy( iam.PolicyStatement(actions=["textract:*"], resources=["*"])) # ------------------------------------------------------------ # Async Jobs Results Processor jobResultProcessor = _lambda.Function( self, "JobResultProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/jobresultprocessor"), handler="lambda_function.lambda_handler", memory_size=2000, reserved_concurrent_executions=50, timeout=core.Duration.seconds(900), environment={ "OUTPUT_TABLE": outputTable.table_name, "DOCUMENTS_TABLE": documentsTable.table_name, "AWS_DATA_PATH": "models", }, ) # Layer jobResultProcessor.add_layers(helperLayer) jobResultProcessor.add_layers(textractorLayer) # Triggers jobResultProcessor.add_event_source( SqsEventSource(jobResultsQueue, batch_size=1)) # Permissions outputTable.grant_read_write_data(jobResultProcessor) documentsTable.grant_read_write_data(jobResultProcessor) contentBucket.grant_read_write(jobResultProcessor) existingContentBucket.grant_read_write(jobResultProcessor) jobResultProcessor.add_to_role_policy( iam.PolicyStatement(actions=["textract:*", "comprehend:*"], resources=["*"])) # -------------- # PDF Generator pdfGenerator = _lambda.Function( self, "PdfGenerator", runtime=_lambda.Runtime.JAVA_8, code=_lambda.Code.from_asset("awscdk/lambda/pdfgenerator"), handler="DemoLambdaV2::handleRequest", memory_size=3000, timeout=core.Duration.seconds(900), ) contentBucket.grant_read_write(pdfGenerator) existingContentBucket.grant_read_write(pdfGenerator) pdfGenerator.grant_invoke(syncProcessor) pdfGenerator.grant_invoke(asyncProcessor)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ddb_whats_new_podcast = _ddb.Table( self, 'whats-new-podcast-table', partition_key={ 'name': 'guid', 'type': _ddb.AttributeType.STRING }, table_name='whats-new-podcast', ) ddb_script = _ddb.Table(self, 'whats-new-podcast-script', partition_key={ 'name': 'date', 'type': _ddb.AttributeType.STRING }, table_name='whats-new-podcast-script') sqs_whats_new_podcast = _sqs.Queue( self, 'whats-new-podcast-queue', queue_name='whats-new-podcast-queue', ) sns_whats_new_podcast = _sns.Topic( self, 'whats-new-podcast-topic', display_name='AWS News', topic_name='whats-new-podcast-topic', ) sns_whats_new_podcast.add_subscription( _sns_subs.SqsSubscription(sqs_whats_new_podcast)) s3_podcast_bucket = _s3.Bucket( self, "whats-new-podcast-bucket", bucket_name= f"whats-new-podcast-bucket-{random.randrange(1000000000)}") statement_sns_publish = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=['sns:Publish'], resources=[sns_whats_new_podcast.topic_arn]) statement_dynamodb = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=['dynamodb:PutItem', 'dynamodb:GetItem'], resources=[ddb_whats_new_podcast.table_arn]) lambda_rss_to_sns = _lambda.Function( self, 'whats-new-podcast-rss-to-sns', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda/rss-to-sns'), handler='rss-to-sns.lambda_handler', environment={ 'DYNAMO_DB_TABLE': ddb_whats_new_podcast.table_name, 'SNS_TOPIC': sns_whats_new_podcast.topic_arn }, function_name='whats-new-podcast-rss-to-sns', timeout=core.Duration.seconds(30), ) lambda_rss_to_sns.add_to_role_policy(statement_dynamodb) lambda_rss_to_sns.add_to_role_policy(statement_sns_publish) rule_rss = _events.Rule(self, 'whats-new-podcast-rss-rule', schedule=_events.Schedule.cron(minute='0/15', hour='1-23/2', month='*', week_day='*', year='*'), rule_name='whats-new-podcast-rss-rule') rule_rss.add_target(_targets.LambdaFunction(lambda_rss_to_sns)) lambda_generate_script = _lambda.Function( self, 'whats-new-podcast-generate-script', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda/generate-script'), handler='generate-script.lambda_handler', environment={'DYNAMO_DB_TABLE': ddb_script.table_name}, function_name='whats-new-podcast-generate-script', timeout=core.Duration.seconds(30), ) statement_lambda_generate_script_ddb = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=[ 'dynamodb:PutItem', 'dynamodb:GetItem', 'dynamodb:Query', 'dynamodb:UpdateItem', ], resources=[ddb_script.table_arn]) lambda_generate_script.add_to_role_policy( statement_lambda_generate_script_ddb) lambda_generate_script_source_queue = _lambda_events.SqsEventSource( queue=sqs_whats_new_podcast, batch_size=1) lambda_generate_script.add_event_source( lambda_generate_script_source_queue) lambda_generate_voice = _lambda.Function( self, 'whats-new-podcast-generate-voice', runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.asset('lambda/generate-voice'), handler='generate-voice.lambda_handler', environment={ 'DYNAMO_DB_TABLE': ddb_script.table_name, 'S3_BUCKET': s3_podcast_bucket.bucket_name }, function_name='whats-new-podcast-generate-voice', timeout=core.Duration.seconds(30), ) statement_lambda_generate_voice_ddb = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=[ 'dynamodb:GetItem', 'dynamodb:Query', ], resources=[ddb_script.table_arn]) statement_lambda_generate_voice_polly = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=[ 'polly:SynthesizeSpeech', 'polly:StartSpeechSynthesisTask' ], resources=['*']) statement_lambda_generate_voice_s3 = _iam.PolicyStatement( effect=_iam.Effect.ALLOW, actions=['s3:PutObject'], resources=[f"{s3_podcast_bucket.bucket_arn}/*"]) lambda_generate_voice.add_to_role_policy( statement_lambda_generate_voice_ddb) lambda_generate_voice.add_to_role_policy( statement_lambda_generate_voice_polly) lambda_generate_voice.add_to_role_policy( statement_lambda_generate_voice_s3) rule_voice = _events.Rule( self, 'whats-new-podcast-generate-voice-rule', schedule=_events.Schedule.cron(minute='30', hour='0', month='*', week_day='*', year='*'), rule_name='whats-new-podcast-generate-voice-rule') rule_voice.add_target(_targets.LambdaFunction(lambda_generate_voice))
def create_all_queues(self) -> None: """ Create all STACK queues, attach subscriptions and alarms """ # General DLQs for lambdas (not API) self.create_queue(id="dead_letter_queue") general_dlq_alarm = cloudwatch.Alarm( self, "DLQAlarm", metric=self.queues_["dead_letter_queue"].metric( "ApproximateNumberOfMessagesVisible"), evaluation_periods=1, threshold=0.0, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, ) general_dlq_alarm.add_alarm_action( cw_actions.SnsAction(self.topics_["alarm_topic"])) # DLQ for API lambdas self.create_queue(id="api_dead_letter_queue") api_dlq_alarm = cloudwatch.Alarm( self, "APIDLQAlarm", metric=self.queues_["api_dead_letter_queue"].metric( "ApproximateNumberOfMessagesVisible"), evaluation_periods=1, threshold=0.0, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, ) api_dlq_alarm.add_alarm_action( cw_actions.SnsAction(self.topics_["alarm_topic"])) # The new_scenes_queue subscribe to CBERS 4/4A quicklooks notifications. The # STAC items are generated from the original INPE metadata file as # soon as the quicklooks are created in the PDS bucket # This code fragment creates the queue, the associated dlq and # subscribe to CBERS 4/4A quicklook notification topics self.create_queue( id="process_new_scenes_queue_dlq", retention_period=core.Duration.seconds(1209600), ) process_new_scenes_queue_alarm = cloudwatch.Alarm( self, "ProcessNewScenesQueueAlarm", metric=self.queues_["process_new_scenes_queue_dlq"].metric( "ApproximateNumberOfMessagesVisible"), evaluation_periods=1, threshold=0.0, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, ) process_new_scenes_queue_alarm.add_alarm_action( cw_actions.SnsAction(self.topics_["alarm_topic"])) self.create_queue( id="new_scenes_queue", visibility_timeout=core.Duration.seconds(385), retention_period=core.Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=1, queue=self.queues_["process_new_scenes_queue_dlq"]), ) # Add subscriptions for each CB4 camera sns.Topic.from_topic_arn( self, id="CB4MUX", topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4MUXQuicklook", ).add_subscription( sns_subscriptions.SqsSubscription( self.queues_["new_scenes_queue"])) sns.Topic.from_topic_arn( self, id="CB4AWFI", topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4AWFIQuicklook", ).add_subscription( sns_subscriptions.SqsSubscription( self.queues_["new_scenes_queue"])) sns.Topic.from_topic_arn( self, id="CB4PAN10M", topic_arn= "arn:aws:sns:us-east-1:599544552497:NewCB4PAN10MQuicklook", ).add_subscription( sns_subscriptions.SqsSubscription( self.queues_["new_scenes_queue"])) sns.Topic.from_topic_arn( self, id="CBPAN5M", topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4PAN5MQuicklook", ).add_subscription( sns_subscriptions.SqsSubscription( self.queues_["new_scenes_queue"])) # Subscription for CB4A (all cameras) sns.Topic.from_topic_arn( self, id="CB4A", topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4AQuicklook", ).add_subscription( sns_subscriptions.SqsSubscription( self.queues_["new_scenes_queue"])) self.create_queue( id="catalog_prefix_update_queue", visibility_timeout=core.Duration.seconds(60), retention_period=core.Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["dead_letter_queue"]), ) # Reconcile queue for INPE's XML metadata self.create_queue( id="consume_reconcile_queue_dlq", retention_period=core.Duration.seconds(1209600), ) consume_reconcile_queue_alarm = cloudwatch.Alarm( self, "ConsumeReconcileQueueAlarm", metric=self.queues_["consume_reconcile_queue_dlq"].metric( "ApproximateNumberOfMessagesVisible"), evaluation_periods=1, threshold=0.0, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, ) consume_reconcile_queue_alarm.add_alarm_action( cw_actions.SnsAction(self.topics_["alarm_topic"])) self.create_queue( id="reconcile_queue", visibility_timeout=core.Duration.seconds(1000), retention_period=core.Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["consume_reconcile_queue_dlq"]), ) # Reconcile queue for STAC items self.create_queue( id="consume_stac_reconcile_queue_dlq", retention_period=core.Duration.seconds(1209600), ) consume_stac_reconcile_queue_alarm = cloudwatch.Alarm( self, "ConsumeStacReconcileQueueAlarm", metric=self.queues_["consume_stac_reconcile_queue_dlq"].metric( "ApproximateNumberOfMessagesVisible"), evaluation_periods=1, threshold=0.0, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, ) consume_stac_reconcile_queue_alarm.add_alarm_action( cw_actions.SnsAction(self.topics_["alarm_topic"])) self.create_queue( id="stac_reconcile_queue", visibility_timeout=core.Duration.seconds(1000), retention_period=core.Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["consume_stac_reconcile_queue_dlq"], ), ) # Queue for STAC items to be inserted into Elasticsearch. Subscribe to the # topic with new stac items self.create_queue( id="insert_into_elasticsearch_queue", visibility_timeout=core.Duration.seconds(180), retention_period=core.Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["dead_letter_queue"]), ) # Subscription for new item topics self.topics_["stac_item_topic"].add_subscription( sns_subscriptions.SqsSubscription( self.queues_["insert_into_elasticsearch_queue"])) # Subscription for reconciled item topics self.topics_["reconcile_stac_item_topic"].add_subscription( sns_subscriptions.SqsSubscription( self.queues_["insert_into_elasticsearch_queue"])) # Backup queue for STAC items inserted into Elasticsearch. # This holds the same items received by "insert_into_elasticsearch_queue", # simply holding them for some time to allow recover from ES # cluster failures (see #78) # This queue subscribe only to new item topics self.create_queue( id="backup_insert_into_elasticsearch_queue", visibility_timeout=core.Duration.seconds(180), retention_period=core.Duration.days( settings.backup_queue_retention_days), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["dead_letter_queue"]), ) # Subscription for new item topics self.topics_["stac_item_topic"].add_subscription( sns_subscriptions.SqsSubscription( self.queues_["backup_insert_into_elasticsearch_queue"]))
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) queue = sqs.Queue( self, "cnfqueue", visibility_timeout=core.Duration.seconds(300), ) topic = sns.Topic(self, "cnftopic") az = "us-east-1a,us-east-1b,us-east-1c" cidr = ec2.CfnVPCCidrBlock(self, "cnfCidr", vpc_id="vpc-0948e89bfc65a24c2", cidr_block="10.124.0.0/22") # print(len(az.split(","))) if len(az.split(",")) == 3: sub1 = ec2.CfnSubnet(self, f"cnfnet1", availability_zone=str(az.split(",")[0]), cidr_block="10.124.0.0/24", vpc_id="vpc-0948e89bfc65a24c2") sub2 = ec2.CfnSubnet(self, f"cnfnet2", availability_zone=str(az.split(",")[1]), cidr_block="10.124.1.0/24", vpc_id="vpc-0948e89bfc65a24c2") sub3 = ec2.CfnSubnet(self, f"cnfnet3", availability_zone=str(az.split(",")[2]), cidr_block="10.124.2.0/25", vpc_id="vpc-0948e89bfc65a24c2") sub1.add_depends_on(cidr) sub2.add_depends_on(cidr) sub3.add_depends_on(cidr) # ec2.CfnSubnetRouteTableAssociation(self, "cnfnet1ass", "rtb-0acd4a7a6ab02e9d3", sub1.logical_id()) else: sub1 = ec2.CfnSubnet(self, f"cnfnet1", availability_zone=str(az.split(",")[0]), cidr_block="10.124.0.0/24", vpc_id="vpc-0948e89bfc65a24c2") sub2 = ec2.CfnSubnet(self, f"cnfnet2", availability_zone=str(az.split(",")[1]), cidr_block="10.124.1.0/24", vpc_id="vpc-0948e89bfc65a24c2") sub1.add_depends_on(cidr) sub2.add_depends_on(cidr) count = 1 ssmparam = ssm.StringParameter(self, f"subnet{count}", allowed_pattern=".*", description="The value Foo", parameter_name=f"FooParameter{count}", string_value="Foo", tier=ssm.ParameterTier.ADVANCED) # co=0 # while co < 3: # repository = ecr.Repository(self, f"Repository{co}") # co=co+1 topic.add_subscription(subs.SqsSubscription(queue))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) ### # SNS Topic Creation # Our API Gateway posts messages directly to this ### topic = sns.Topic(self, 'theBigFanTopic', display_name='The Big Fan CDK Pattern Topic') ### # SQS Subscribers creation for our SNS Topic # 2 subscribers, one for messages with a status of created one for any other message ### # Status:created SNS Subscriber Queue created_status_queue = sqs.Queue( self, 'BigFanTopicStatusCreatedSubscriberQueue', visibility_timeout=core.Duration.seconds(300), queue_name='BigFanTopicStatusCreatedSubscriberQueue') # Only send messages to our created_status_queue with a status of created created_filter = sns.SubscriptionFilter.string_filter( whitelist=['created']) topic.add_subscription( subscriptions.SqsSubscription( created_status_queue, raw_message_delivery=True, filter_policy={'status': created_filter})) # Any other status SNS Subscriber Queue other_status_queue = sqs.Queue( self, 'BigFanTopicAnyOtherStatusSubscriberQueue', visibility_timeout=core.Duration.seconds(300), queue_name='BigFanTopicAnyOtherStatusSubscriberQueue') # Only send messages to our other_status_queue that do not have a status of created other_filter = sns.SubscriptionFilter.string_filter( blacklist=['created']) topic.add_subscription( subscriptions.SqsSubscription( other_status_queue, raw_message_delivery=True, filter_policy={'status': other_filter})) ### # Creation of Lambdas that subscribe to above SQS queues ### # Created status queue lambda sqs_created_status_subscriber = _lambda.Function( self, "SQSCreatedStatusSubscribeLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="createdStatus.handler", code=_lambda.Code.from_asset("lambda_fns/subscribe")) created_status_queue.grant_consume_messages( sqs_created_status_subscriber) sqs_created_status_subscriber.add_event_source( _event.SqsEventSource(created_status_queue)) # Any other status queue lambda sqs_other_status_subscriber = _lambda.Function( self, "SQSAnyOtherStatusSubscribeLambdaHandler", runtime=_lambda.Runtime.NODEJS_12_X, handler="anyOtherStatus.handler", code=_lambda.Code.from_asset("lambda_fns/subscribe")) other_status_queue.grant_consume_messages(sqs_other_status_subscriber) sqs_other_status_subscriber.add_event_source( _event.SqsEventSource(other_status_queue)) ### # API Gateway Creation # This is complicated because it transforms the incoming json payload into a query string url # this url is used to post the payload to sns without a lambda inbetween ### gateway = api_gw.RestApi( self, 'theBigFanAPI', deploy_options=api_gw.StageOptions( metrics_enabled=True, logging_level=api_gw.MethodLoggingLevel.INFO, data_trace_enabled=True, stage_name='prod')) # Give our gateway permissions to interact with SNS api_gw_sns_role = iam.Role( self, 'DefaultLambdaHanderRole', assumed_by=iam.ServicePrincipal('apigateway.amazonaws.com')) topic.grant_publish(api_gw_sns_role) # shortening the lines of later code schema = api_gw.JsonSchema schema_type = api_gw.JsonSchemaType # Because this isn't a proxy integration, we need to define our response model response_model = gateway.add_model( 'ResponseModel', content_type='application/json', model_name='ResponseModel', schema=schema( schema=api_gw.JsonSchemaVersion.DRAFT4, title='pollResponse', type=schema_type.OBJECT, properties={'message': schema(type=schema_type.STRING)})) error_response_model = gateway.add_model( 'ErrorResponseModel', content_type='application/json', model_name='ErrorResponseModel', schema=schema(schema=api_gw.JsonSchemaVersion.DRAFT4, title='errorResponse', type=schema_type.OBJECT, properties={ 'state': schema(type=schema_type.STRING), 'message': schema(type=schema_type.STRING) })) request_template = "Action=Publish&" + \ "TargetArn=$util.urlEncode('" + topic.topic_arn + "')&" + \ "Message=$util.urlEncode($input.path('$.message'))&" + \ "Version=2010-03-31&" + \ "MessageAttributes.entry.1.Name=status&" + \ "MessageAttributes.entry.1.Value.DataType=String&" + \ "MessageAttributes.entry.1.Value.StringValue=$util.urlEncode($input.path('$.status'))" # This is the VTL to transform the error response error_template = { "state": 'error', "message": "$util.escapeJavaScript($input.path('$.errorMessage'))" } error_template_string = json.dumps(error_template, separators=(',', ':')) # This is how our gateway chooses what response to send based on selection_pattern integration_options = api_gw.IntegrationOptions( credentials_role=api_gw_sns_role, request_parameters={ 'integration.request.header.Content-Type': "'application/x-www-form-urlencoded'" }, request_templates={"application/json": request_template}, passthrough_behavior=api_gw.PassthroughBehavior.NEVER, integration_responses=[ api_gw.IntegrationResponse( status_code='200', response_templates={ "application/json": json.dumps({"message": 'message added to topic'}) }), api_gw.IntegrationResponse( selection_pattern="^\[Error\].*", status_code='400', response_templates={ "application/json": error_template_string }, response_parameters={ 'method.response.header.Content-Type': "'application/json'", 'method.response.header.Access-Control-Allow-Origin': "'*'", 'method.response.header.Access-Control-Allow-Credentials': "'true'" }) ]) # Add an SendEvent endpoint onto the gateway gateway.root.add_resource('SendEvent') \ .add_method('POST', api_gw.Integration(type=api_gw.IntegrationType.AWS, integration_http_method='POST', uri='arn:aws:apigateway:us-east-1:sns:path//', options=integration_options ), method_responses=[ api_gw.MethodResponse(status_code='200', response_parameters={ 'method.response.header.Content-Type': True, 'method.response.header.Access-Control-Allow-Origin': True, 'method.response.header.Access-Control-Allow-Credentials': True }, response_models={ 'application/json': response_model }), api_gw.MethodResponse(status_code='400', response_parameters={ 'method.response.header.Content-Type': True, 'method.response.header.Access-Control-Allow-Origin': True, 'method.response.header.Access-Control-Allow-Credentials': True }, response_models={ 'application/json': error_response_model }), ] )