def _create_lambdas(self): clean_pycache() for root, dirs, files in os.walk(LAMBDAS_DIR): for f in files: if f != "__init__.py": continue parent_folder = os.path.basename(os.path.dirname(root)) lambda_folder = os.path.basename(root) name = f"{parent_folder}-{lambda_folder}" lambda_config = self.lambdas_config[name] layers = [] for layer_name in lambda_config["layers"]: layers.append(self.layers[layer_name]) lambda_role = Role( self, f"{name}_role", assumed_by=ServicePrincipal(service="lambda.amazonaws.com") ) for policy in lambda_config["policies"]: lambda_role.add_to_policy(policy) lambda_role.add_managed_policy( ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole")) lambda_args = { "code": Code.from_asset(root), "handler": "__init__.handle", "runtime": Runtime.PYTHON_3_8, "layers": layers, "function_name": name, "environment": lambda_config["variables"], "role": lambda_role, "timeout": Duration.seconds(lambda_config["timeout"]), "memory_size": lambda_config["memory"], } if "concurrent_executions" in lambda_config: lambda_args["reserved_concurrent_executions"] = lambda_config["concurrent_executions"] self.lambdas[name] = Function(self, name, **lambda_args) self.lambdas["sqs_handlers-post_anime"].add_event_source(SqsEventSource(self.post_anime_queue)) Rule( self, "titles_updater", schedule=Schedule.cron(hour="2", minute="10"), targets=[LambdaFunction(self.lambdas["crons-titles_updater"])] ) Rule( self, "episodes_updater", schedule=Schedule.cron(hour="4", minute="10"), targets=[LambdaFunction(self.lambdas["crons-episodes_updater"])] )
async def create_canary_function(self, id: str) -> Function: function = None with open('canary/canary.py', 'r') as code: canary_code = code.read() function = Function( self, '{}CanaryFunction'.format(id), timeout=Duration.seconds(3), code=InlineCode(canary_code), handler='index.handler', tracing=Tracing.ACTIVE, initial_policy=[MINIMAL_FUNCTION_POLICY_STATEMENT], runtime=Runtime( name='python3.7', supports_inline_code=True, ) ) Rule(self, '{}CanaryRule'.format(id), enabled=True, schedule=Schedule.cron(), targets=[LambdaFunction(handler=function)]) return function
def __init__(self, app: App, id: str, txt: str, env: dict, policies: PolicyStatement, domain: str, hosted_zone_id: str) -> None: super().__init__(app, id) env['HOSTED_ZONE_ID'] = hosted_zone_id self.function = SingletonFunction(self, '{}Function'.format('{}'.format(id)), uuid=str(uuid4()), code=Code.inline(txt), runtime=Runtime( 'python3.7', supports_inline_code=True), handler='index.handler', environment=env) policy = Policy(self, '{}Policy'.format(id)) self.function.role.attach_inline_policy(policy) policy.add_statements(policies) rule_target = LambdaFunction(self.function) current_time = datetime.now() run_time = current_time + timedelta(minutes=3) run_schedule = Schedule.cron(year=str(run_time.year), month=str(run_time.month), day=str(run_time.day), hour=str(run_time.hour), minute=str(run_time.minute)) self.rule = Rule(self, '{}Rule'.format(id), enabled=True, schedule=run_schedule, targets=[rule_target])
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) table_name = "posts2" function_name = "cl2" email = "*****@*****.**" table = Table( self, "cl_posts", table_name=table_name, partition_key=Attribute(name="url", type=AttributeType.STRING), time_to_live_attribute="ttl", ) function = PythonFunction( self, "cl_function", function_name=function_name, entry="src", index="app.py", runtime=Runtime.PYTHON_3_8, environment={ "cl_email": email, "cl_table_name": table_name }, timeout=Duration.seconds(300), initial_policy=[ PolicyStatement( actions=["ses:SendEmail", "ses:VerifyEmailIdentity"], resources=[ f"arn:aws:ses:{self.region}:{self.account}:identity/{email}" ], ), PolicyStatement( actions=[ "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem" ], resources=[table.table_arn], ), ], ) with open("events/event.json") as f: event = json.load(f) Rule( self, "cl_schedule", schedule=Schedule.expression("cron(0 19 * * ? *)"), targets=[ LambdaFunction(function, event=RuleTargetInput.from_object(event)) ], )
def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) self.event_bus = EventBus(scope=self, id='CustomEventBus', event_bus_name='CustomEventBus') self.source = Function( scope=self, id=f'SourceFunction', function_name=f'SourceFunction', code=Code.from_asset(path='./code_source/'), handler='index.handler', runtime=Runtime.PYTHON_3_6, ) self.source.add_to_role_policy(statement=PolicyStatement( actions=['events:PutEvents'], resources=[self.event_bus.event_bus_arn])) """ Define rule. """ self.rule = Rule( scope=self, id='EventBusRule', description='Sample description.', enabled=True, event_bus=self.event_bus, event_pattern=EventPattern(detail={ 'Domain': ["MedInfo"], 'Reason': ["InvokeTarget"] }), rule_name='EventBusRule', ) """ Add target. """ self.target = Function( scope=self, id=f'TargetFunction', function_name=f'TargetFunction', code=Code.from_asset(path='./code_target/'), handler='index.handler', runtime=Runtime.PYTHON_3_6, ) self.target: Union[IRuleTarget, LambdaFunction] = LambdaFunction( handler=self.target) self.rule.add_target(target=self.target)
def _create_lambdas(self): for root, dirs, files in os.walk(LAMBDAS_DIR): for f in files: if f != "__init__.py": continue parent_folder = os.path.basename(os.path.dirname(root)) lambda_folder = os.path.basename(root) name = f"{parent_folder}-{lambda_folder}" lambda_config = self.lambdas_config[name] layers = [] for layer_name in lambda_config["layers"]: layers.append(self.layers[layer_name]) lambda_role = Role(self, f"{name}_role", assumed_by=ServicePrincipal( service="lambda.amazonaws.com")) for policy in lambda_config["policies"]: lambda_role.add_to_policy(policy) lambda_role.add_managed_policy( ManagedPolicy.from_aws_managed_policy_name( "service-role/AWSLambdaBasicExecutionRole")) self.lambdas[name] = Function( self, name, code=Code.from_asset(root), handler="__init__.handle", runtime=Runtime.PYTHON_3_8, layers=layers, function_name=name, environment=lambda_config["variables"], role=lambda_role, timeout=Duration.seconds(lambda_config["timeout"]), memory_size=lambda_config["memory"], ) Rule(self, "update_eps", schedule=Schedule.cron(hour="2", minute="10"), targets=[LambdaFunction(self.lambdas["cron-update_eps"])])
def create_ecs_lambda(self, cluster: ICluster, auto_scaling_group: AutoScalingGroup): lambda_func = Function( self, "LambdaECS", code=Code.from_asset("./lambdas/nlb-ecs"), handler="index.lambda_handler", runtime=Runtime.PYTHON_3_8, timeout=Duration.seconds(30), environment={ "AUTO_SCALING_GROUP_NAME": auto_scaling_group.auto_scaling_group_name, }, ) lambda_func.add_to_role_policy( PolicyStatement( actions=[ "autoscaling:DescribeAutoScalingGroups", "ssm:SendCommand", "ssm:GetCommandInvocation", ], resources=[ "*", ], )) Rule( self, "ECS", event_pattern=EventPattern( detail_type=["ECS Task State Change"], detail={ "clusterArn": [cluster.cluster_arn], }, source=["aws.ecs"], ), targets=[LambdaFunction(lambda_func)], )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) mytopic = sns.Topic( self, "BillingAlert" ) email_parameter = core.CfnParameter(self, "email-param") dailyBudget_parameter = core.CfnParameter(self, "DailyBudget") monthlyGrowthRate_parameter = core.CfnParameter(self, "MonthlyGrowthRate") S3CodePath_parameter = core.CfnParameter(self, "S3CodePath") emailAddress = getattr(email_parameter,"value_as_string") dailyBudget_value = getattr(dailyBudget_parameter,"value_as_string") monthlyGrowthRate_value = getattr(monthlyGrowthRate_parameter,"value_as_string") mytopic.add_subscription(subscriptions.EmailSubscription(emailAddress)) myrole = iam.Role(self, "BillianAlertRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com")) myrole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSNSFullAccess")) myrole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AWSLambdaBasicExecutionRole")) myrole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("CloudWatchFullAccess ")) function = awslambda.Function(self, "MyLambda", code=awslambda.Code.from_cfn_parameters(object_key_param=S3CodePath_parameter), handler="lambda_function.py", runtime=awslambda.Runtime.PYTHON_3_7, role = myrole, function_name= "BillingAlert", memory_size= 3000 ) function.add_environment("DailyBudget", dailyBudget_value) function.add_environment("MonthlyGrowthRate", monthlyGrowthRate_value) function.add_environment("SNSARN", getattr(mytopic,"topic_arn")) targetFunction = LambdaFunction(function) Rule(self, "ScheduleRuleForBillingAlert", schedule=Schedule.cron(minute="0", hour="4"), targets=[targetFunction] )
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: super().__init__(scope, id, **kwargs) # The code that defines your stack goes here # ********* SNS Topics ************* jobCompletionTopic = sns.Topic(self, "JobCompletion") # **********IAM Roles****************************** textractServiceRole = iam.Role( self, "TextractServiceRole", assumed_by=iam.ServicePrincipal("textract.amazonaws.com"), ) textractServiceRole.add_to_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=[jobCompletionTopic.topic_arn], actions=["sns:Publish"], )) comprehendServiceRole = iam.Role( self, "ComprehendServiceRole", assumed_by=iam.ServicePrincipal("comprehend.amazonaws.com"), ) comprehendServiceRole.add_to_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, resources=["*"], actions=[ "comprehend:*", "s3:ListAllMyBuckets", "s3:ListBucket", "s3:GetBucketLocation", "iam:ListRoles", "iam:GetRole", ], )) # **********S3 Batch Operations Role****************************** s3BatchOperationsRole = iam.Role( self, "S3BatchOperationsRole", assumed_by=iam.ServicePrincipal( "batchoperations.s3.amazonaws.com"), ) # **********S3 Bucket****************************** # S3 bucket for input documents and output contentBucket = s3.Bucket(self, "DocumentsBucket", versioned=False) existingContentBucket = s3.Bucket(self, "ExistingDocumentsBucket", versioned=False) existingContentBucket.grant_read_write(s3BatchOperationsRole) inventoryAndLogsBucket = s3.Bucket(self, "InventoryAndLogsBucket", versioned=False) inventoryAndLogsBucket.grant_read_write(s3BatchOperationsRole) # **********DynamoDB Table************************* # DynamoDB table with links to output in S3 outputTable = dynamodb.Table( self, "OutputTable", partition_key={ "name": "documentId", "type": dynamodb.AttributeType.STRING, }, sort_key={ "name": "outputType", "type": dynamodb.AttributeType.STRING, }, ) # DynamoDB table with links to output in S3 documentsTable = dynamodb.Table( self, "DocumentsTable", partition_key={ "name": "documentId", "type": dynamodb.AttributeType.STRING, }, stream=dynamodb.StreamViewType.NEW_IMAGE, ) # **********SQS Queues***************************** # DLQ (Dead Letter Queue) dlq = sqs.Queue( self, "DLQ", visibility_timeout=core.Duration.seconds(30), retention_period=core.Duration.seconds(1209600), ) # Input Queue for sync jobs syncJobsQueue = sqs.Queue( self, "SyncJobs", visibility_timeout=core.Duration.seconds(30), retention_period=core.Duration.seconds(1209600), dead_letter_queue={ "queue": dlq, "max_receive_count": 50 }, ) # Input Queue for async jobs asyncJobsQueue = sqs.Queue( self, "AsyncJobs", visibility_timeout=core.Duration.seconds(30), retention_period=core.Duration.seconds(1209600), dead_letter_queue={ "queue": dlq, "max_receive_count": 50 }, ) # Queue jobResultsQueue = sqs.Queue( self, "JobResults", visibility_timeout=core.Duration.seconds(900), retention_period=core.Duration.seconds(1209600), dead_letter_queue={ "queue": dlq, "max_receive_count": 50 }, ) # Trigger # jobCompletionTopic.subscribeQueue(jobResultsQueue) jobCompletionTopic.add_subscription( snsSubscriptions.SqsSubscription(jobResultsQueue)) # **********Lambda Functions****************************** # Helper Layer with helper functions helperLayer = _lambda.LayerVersion( self, "HelperLayer", code=_lambda.Code.from_asset("awscdk/lambda/helper"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], license="Apache-2.0", description="Helper layer.", ) # Textractor helper layer textractorLayer = _lambda.LayerVersion( self, "Textractor", code=_lambda.Code.from_asset("awscdk/lambda/textractor"), compatible_runtimes=[_lambda.Runtime.PYTHON_3_7], license="Apache-2.0", description="Textractor layer.", ) # ----------------------------------------------------------- # S3 Event processor s3Processor = _lambda.Function( self, "S3Processor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/s3processor"), handler="lambda_function.lambda_handler", environment={ "SYNC_QUEUE_URL": syncJobsQueue.queue_url, "ASYNC_QUEUE_URL": asyncJobsQueue.queue_url, "DOCUMENTS_TABLE": documentsTable.table_name, "OUTPUT_TABLE": outputTable.table_name, }, ) # Layer s3Processor.add_layers(helperLayer) # Trigger s3Processor.add_event_source( S3EventSource(contentBucket, events=[s3.EventType.OBJECT_CREATED])) # Permissions documentsTable.grant_read_write_data(s3Processor) syncJobsQueue.grant_send_messages(s3Processor) asyncJobsQueue.grant_send_messages(s3Processor) # ------------------------------------------------------------ # S3 Batch Operations Event processor s3BatchProcessor = _lambda.Function( self, "S3BatchProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/s3batchprocessor"), handler="lambda_function.lambda_handler", environment={ "DOCUMENTS_TABLE": documentsTable.table_name, "OUTPUT_TABLE": outputTable.table_name, }, reserved_concurrent_executions=1, ) # Layer s3BatchProcessor.add_layers(helperLayer) # Permissions documentsTable.grant_read_write_data(s3BatchProcessor) s3BatchProcessor.grant_invoke(s3BatchOperationsRole) s3BatchOperationsRole.add_to_policy( iam.PolicyStatement(actions=["lambda:*"], resources=["*"])) # ------------------------------------------------------------ # Document processor (Router to Sync/Async Pipeline) documentProcessor = _lambda.Function( self, "TaskProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/documentprocessor"), handler="lambda_function.lambda_handler", environment={ "SYNC_QUEUE_URL": syncJobsQueue.queue_url, "ASYNC_QUEUE_URL": asyncJobsQueue.queue_url, }, ) # Layer documentProcessor.add_layers(helperLayer) # Trigger documentProcessor.add_event_source( DynamoEventSource( documentsTable, starting_position=_lambda.StartingPosition.TRIM_HORIZON, )) # Permissions documentsTable.grant_read_write_data(documentProcessor) syncJobsQueue.grant_send_messages(documentProcessor) asyncJobsQueue.grant_send_messages(documentProcessor) # ------------------------------------------------------------ # Sync Jobs Processor (Process jobs using sync APIs) syncProcessor = _lambda.Function( self, "SyncProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/documentprocessor"), handler="lambda_function.lambda_handler", environment={ "OUTPUT_TABLE": outputTable.table_name, "DOCUMENTS_TABLE": documentsTable.table_name, "AWS_DATA_PATH": "models", }, reserved_concurrent_executions=1, timeout=core.Duration.seconds(25), ) # Layer syncProcessor.add_layers(helperLayer) syncProcessor.add_layers(textractorLayer) # Trigger syncProcessor.add_event_source( SqsEventSource(syncJobsQueue, batch_size=1)) # Permissions contentBucket.grant_read_write(syncProcessor) existingContentBucket.grant_read_write(syncProcessor) outputTable.grant_read_write_data(syncProcessor) documentsTable.grant_read_write_data(syncProcessor) syncProcessor.add_to_role_policy( iam.PolicyStatement(actions=["textract:*"], resources=["*"])) # ------------------------------------------------------------ # Async Job Processor (Start jobs using Async APIs) asyncProcessor = _lambda.Function( self, "ASyncProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/asyncprocessor"), handler="lambda_function.lambda_handler", environment={ "ASYNC_QUEUE_URL": asyncJobsQueue.queue_url, "SNS_TOPIC_ARN": jobCompletionTopic.topic_arn, "SNS_ROLE_ARN": textractServiceRole.role_arn, "AWS_DATA_PATH": "models", }, reserved_concurrent_executions=1, timeout=core.Duration.seconds(60), ) # asyncProcessor.addEnvironment("SNS_TOPIC_ARN", textractServiceRole.topic_arn) # Layer asyncProcessor.add_layers(helperLayer) # Triggers # Run async job processor every 5 minutes # Enable code below after test deploy rule = events.Rule( self, "Rule", schedule=events.Schedule.expression("rate(2 minutes)")) rule.add_target(LambdaFunction(asyncProcessor)) # Run when a job is successfully complete asyncProcessor.add_event_source(SnsEventSource(jobCompletionTopic)) # Permissions contentBucket.grant_read(asyncProcessor) existingContentBucket.grant_read_write(asyncProcessor) asyncJobsQueue.grant_consume_messages(asyncProcessor) asyncProcessor.add_to_role_policy( iam.PolicyStatement( actions=["iam:PassRole"], resources=[textractServiceRole.role_arn], )) asyncProcessor.add_to_role_policy( iam.PolicyStatement(actions=["textract:*"], resources=["*"])) # ------------------------------------------------------------ # Async Jobs Results Processor jobResultProcessor = _lambda.Function( self, "JobResultProcessor", runtime=_lambda.Runtime.PYTHON_3_7, code=_lambda.Code.from_asset("awscdk/lambda/jobresultprocessor"), handler="lambda_function.lambda_handler", memory_size=2000, reserved_concurrent_executions=50, timeout=core.Duration.seconds(900), environment={ "OUTPUT_TABLE": outputTable.table_name, "DOCUMENTS_TABLE": documentsTable.table_name, "AWS_DATA_PATH": "models", }, ) # Layer jobResultProcessor.add_layers(helperLayer) jobResultProcessor.add_layers(textractorLayer) # Triggers jobResultProcessor.add_event_source( SqsEventSource(jobResultsQueue, batch_size=1)) # Permissions outputTable.grant_read_write_data(jobResultProcessor) documentsTable.grant_read_write_data(jobResultProcessor) contentBucket.grant_read_write(jobResultProcessor) existingContentBucket.grant_read_write(jobResultProcessor) jobResultProcessor.add_to_role_policy( iam.PolicyStatement(actions=["textract:*", "comprehend:*"], resources=["*"])) # -------------- # PDF Generator pdfGenerator = _lambda.Function( self, "PdfGenerator", runtime=_lambda.Runtime.JAVA_8, code=_lambda.Code.from_asset("awscdk/lambda/pdfgenerator"), handler="DemoLambdaV2::handleRequest", memory_size=3000, timeout=core.Duration.seconds(900), ) contentBucket.grant_read_write(pdfGenerator) existingContentBucket.grant_read_write(pdfGenerator) pdfGenerator.grant_invoke(syncProcessor) pdfGenerator.grant_invoke(asyncProcessor)
def create_event_handling( self, secrets: List[secretsmanager.Secret], slack_host_ssm_name: str, slack_webhook_ssm_name: str, ) -> lambda_.Function: """ Args: secrets: a list of secrets that we will track for events slack_host_ssm_name: the SSM parameter name for the slack host slack_webhook_ssm_name: the SSM parameter name for the slack webhook id Returns: a lambda event handler """ dirname = os.path.dirname(__file__) filename = os.path.join(dirname, "runtime/notify_slack") env = { # for the moment we don't parametrise at the CDK level.. only needed if this is liable to change "SLACK_HOST_SSM_NAME": slack_host_ssm_name, "SLACK_WEBHOOK_SSM_NAME": slack_webhook_ssm_name, } notifier = lambda_.Function( self, "NotifySlack", runtime=lambda_.Runtime.PYTHON_3_8, code=lambda_.AssetCode(filename), handler="lambda_entrypoint.main", timeout=Duration.minutes(1), environment=env, ) get_ssm_policy = PolicyStatement() # there is some weirdness around SSM parameter ARN formation and leading slashes.. can't be bothered # looking into right now - as the ones we want to use do a have a leading slash # but put in this exception in case if not slack_webhook_ssm_name.startswith( "/") or not slack_host_ssm_name.startswith("/"): raise Exception( "SSM parameters need to start with a leading slash") # see here - the *required* slash between parameter and the actual name uses the leading slash from the actual # name itself.. which is wrong.. get_ssm_policy.add_resources( f"arn:aws:ssm:*:*:parameter{slack_host_ssm_name}") get_ssm_policy.add_resources( f"arn:aws:ssm:*:*:parameter{slack_webhook_ssm_name}") get_ssm_policy.add_actions("ssm:GetParameter") notifier.add_to_role_policy(get_ssm_policy) # we want a rule that traps all the rotation failures for our JWT secrets rule = Rule( self, "NotifySlackRule", ) rule.add_event_pattern( source=["aws.secretsmanager"], detail={ # at the moment only interested in these - add extra events into this array if wanting more "eventName": ["RotationFailed", "RotationSucceeded"], "additionalEventData": { "SecretId": list(map(lambda s: s.secret_arn, secrets)) }, }, ) rule.add_target(LambdaFunction(notifier)) return notifier