def _create_lambdas(self):
clean_pycache()
for root, dirs, files in os.walk(LAMBDAS_DIR):
for f in files:
if f != "__init__.py":
continue
parent_folder = os.path.basename(os.path.dirname(root))
lambda_folder = os.path.basename(root)
name = f"{parent_folder}-{lambda_folder}"
lambda_config = self.lambdas_config[name]
layers = []
for layer_name in lambda_config["layers"]:
layers.append(self.layers[layer_name])
lambda_role = Role(
self,
f"{name}_role",
assumed_by=ServicePrincipal(service="lambda.amazonaws.com")
)
for policy in lambda_config["policies"]:
lambda_role.add_to_policy(policy)
lambda_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"))
lambda_args = {
"code": Code.from_asset(root),
"handler": "__init__.handle",
"runtime": Runtime.PYTHON_3_8,
"layers": layers,
"function_name": name,
"environment": lambda_config["variables"],
"role": lambda_role,
"timeout": Duration.seconds(lambda_config["timeout"]),
"memory_size": lambda_config["memory"],
}
if "concurrent_executions" in lambda_config:
lambda_args["reserved_concurrent_executions"] = lambda_config["concurrent_executions"]
self.lambdas[name] = Function(self, name, **lambda_args)
self.lambdas["sqs_handlers-post_anime"].add_event_source(SqsEventSource(self.post_anime_queue))
Rule(
self,
"titles_updater",
schedule=Schedule.cron(hour="2", minute="10"),
targets=[LambdaFunction(self.lambdas["crons-titles_updater"])]
)
Rule(
self,
"episodes_updater",
schedule=Schedule.cron(hour="4", minute="10"),
targets=[LambdaFunction(self.lambdas["crons-episodes_updater"])]
)
async def create_canary_function(self, id: str) -> Function:
function = None
with open('canary/canary.py', 'r') as code:
canary_code = code.read()
function = Function(
self,
'{}CanaryFunction'.format(id),
timeout=Duration.seconds(3),
code=InlineCode(canary_code),
handler='index.handler',
tracing=Tracing.ACTIVE,
initial_policy=[MINIMAL_FUNCTION_POLICY_STATEMENT],
runtime=Runtime(
name='python3.7',
supports_inline_code=True,
)
)
Rule(self,
'{}CanaryRule'.format(id),
enabled=True,
schedule=Schedule.cron(),
targets=[LambdaFunction(handler=function)])
return function
def __init__(self, app: App, id: str, txt: str, env: dict,
policies: PolicyStatement, domain: str,
hosted_zone_id: str) -> None:
super().__init__(app, id)
env['HOSTED_ZONE_ID'] = hosted_zone_id
self.function = SingletonFunction(self,
'{}Function'.format('{}'.format(id)),
uuid=str(uuid4()),
code=Code.inline(txt),
runtime=Runtime(
'python3.7',
supports_inline_code=True),
handler='index.handler',
environment=env)
policy = Policy(self, '{}Policy'.format(id))
self.function.role.attach_inline_policy(policy)
policy.add_statements(policies)
rule_target = LambdaFunction(self.function)
current_time = datetime.now()
run_time = current_time + timedelta(minutes=3)
run_schedule = Schedule.cron(year=str(run_time.year),
month=str(run_time.month),
day=str(run_time.day),
hour=str(run_time.hour),
minute=str(run_time.minute))
self.rule = Rule(self,
'{}Rule'.format(id),
enabled=True,
schedule=run_schedule,
targets=[rule_target])
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
table_name = "posts2"
function_name = "cl2"
email = "*****@*****.**"
table = Table(
self,
"cl_posts",
table_name=table_name,
partition_key=Attribute(name="url", type=AttributeType.STRING),
time_to_live_attribute="ttl",
)
function = PythonFunction(
self,
"cl_function",
function_name=function_name,
entry="src",
index="app.py",
runtime=Runtime.PYTHON_3_8,
environment={
"cl_email": email,
"cl_table_name": table_name
},
timeout=Duration.seconds(300),
initial_policy=[
PolicyStatement(
actions=["ses:SendEmail", "ses:VerifyEmailIdentity"],
resources=[
f"arn:aws:ses:{self.region}:{self.account}:identity/{email}"
],
),
PolicyStatement(
actions=[
"dynamodb:BatchGetItem", "dynamodb:BatchWriteItem"
],
resources=[table.table_arn],
),
],
)
with open("events/event.json") as f:
event = json.load(f)
Rule(
self,
"cl_schedule",
schedule=Schedule.expression("cron(0 19 * * ? *)"),
targets=[
LambdaFunction(function,
event=RuleTargetInput.from_object(event))
],
)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
self.event_bus = EventBus(scope=self,
id='CustomEventBus',
event_bus_name='CustomEventBus')
self.source = Function(
scope=self,
id=f'SourceFunction',
function_name=f'SourceFunction',
code=Code.from_asset(path='./code_source/'),
handler='index.handler',
runtime=Runtime.PYTHON_3_6,
)
self.source.add_to_role_policy(statement=PolicyStatement(
actions=['events:PutEvents'],
resources=[self.event_bus.event_bus_arn]))
"""
Define rule.
"""
self.rule = Rule(
scope=self,
id='EventBusRule',
description='Sample description.',
enabled=True,
event_bus=self.event_bus,
event_pattern=EventPattern(detail={
'Domain': ["MedInfo"],
'Reason': ["InvokeTarget"]
}),
rule_name='EventBusRule',
)
"""
Add target.
"""
self.target = Function(
scope=self,
id=f'TargetFunction',
function_name=f'TargetFunction',
code=Code.from_asset(path='./code_target/'),
handler='index.handler',
runtime=Runtime.PYTHON_3_6,
)
self.target: Union[IRuleTarget, LambdaFunction] = LambdaFunction(
handler=self.target)
self.rule.add_target(target=self.target)
def _create_lambdas(self):
for root, dirs, files in os.walk(LAMBDAS_DIR):
for f in files:
if f != "__init__.py":
continue
parent_folder = os.path.basename(os.path.dirname(root))
lambda_folder = os.path.basename(root)
name = f"{parent_folder}-{lambda_folder}"
lambda_config = self.lambdas_config[name]
layers = []
for layer_name in lambda_config["layers"]:
layers.append(self.layers[layer_name])
lambda_role = Role(self,
f"{name}_role",
assumed_by=ServicePrincipal(
service="lambda.amazonaws.com"))
for policy in lambda_config["policies"]:
lambda_role.add_to_policy(policy)
lambda_role.add_managed_policy(
ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole"))
self.lambdas[name] = Function(
self,
name,
code=Code.from_asset(root),
handler="__init__.handle",
runtime=Runtime.PYTHON_3_8,
layers=layers,
function_name=name,
environment=lambda_config["variables"],
role=lambda_role,
timeout=Duration.seconds(lambda_config["timeout"]),
memory_size=lambda_config["memory"],
)
Rule(self,
"update_eps",
schedule=Schedule.cron(hour="2", minute="10"),
targets=[LambdaFunction(self.lambdas["cron-update_eps"])])
def create_ecs_lambda(self, cluster: ICluster,
auto_scaling_group: AutoScalingGroup):
lambda_func = Function(
self,
"LambdaECS",
code=Code.from_asset("./lambdas/nlb-ecs"),
handler="index.lambda_handler",
runtime=Runtime.PYTHON_3_8,
timeout=Duration.seconds(30),
environment={
"AUTO_SCALING_GROUP_NAME":
auto_scaling_group.auto_scaling_group_name,
},
)
lambda_func.add_to_role_policy(
PolicyStatement(
actions=[
"autoscaling:DescribeAutoScalingGroups",
"ssm:SendCommand",
"ssm:GetCommandInvocation",
],
resources=[
"*",
],
))
Rule(
self,
"ECS",
event_pattern=EventPattern(
detail_type=["ECS Task State Change"],
detail={
"clusterArn": [cluster.cluster_arn],
},
source=["aws.ecs"],
),
targets=[LambdaFunction(lambda_func)],
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
mytopic = sns.Topic(
self, "BillingAlert"
)
email_parameter = core.CfnParameter(self, "email-param")
dailyBudget_parameter = core.CfnParameter(self, "DailyBudget")
monthlyGrowthRate_parameter = core.CfnParameter(self, "MonthlyGrowthRate")
S3CodePath_parameter = core.CfnParameter(self, "S3CodePath")
emailAddress = getattr(email_parameter,"value_as_string")
dailyBudget_value = getattr(dailyBudget_parameter,"value_as_string")
monthlyGrowthRate_value = getattr(monthlyGrowthRate_parameter,"value_as_string")
mytopic.add_subscription(subscriptions.EmailSubscription(emailAddress))
myrole = iam.Role(self, "BillianAlertRole", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"))
myrole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSNSFullAccess"))
myrole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AWSLambdaBasicExecutionRole"))
myrole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("CloudWatchFullAccess "))
function = awslambda.Function(self, "MyLambda",
code=awslambda.Code.from_cfn_parameters(object_key_param=S3CodePath_parameter),
handler="lambda_function.py",
runtime=awslambda.Runtime.PYTHON_3_7,
role = myrole,
function_name= "BillingAlert",
memory_size= 3000
)
function.add_environment("DailyBudget", dailyBudget_value)
function.add_environment("MonthlyGrowthRate", monthlyGrowthRate_value)
function.add_environment("SNSARN", getattr(mytopic,"topic_arn"))
targetFunction = LambdaFunction(function)
Rule(self, "ScheduleRuleForBillingAlert",
schedule=Schedule.cron(minute="0", hour="4"),
targets=[targetFunction]
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
# ********* SNS Topics *************
jobCompletionTopic = sns.Topic(self, "JobCompletion")
# **********IAM Roles******************************
textractServiceRole = iam.Role(
self,
"TextractServiceRole",
assumed_by=iam.ServicePrincipal("textract.amazonaws.com"),
)
textractServiceRole.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=[jobCompletionTopic.topic_arn],
actions=["sns:Publish"],
))
comprehendServiceRole = iam.Role(
self,
"ComprehendServiceRole",
assumed_by=iam.ServicePrincipal("comprehend.amazonaws.com"),
)
comprehendServiceRole.add_to_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=["*"],
actions=[
"comprehend:*",
"s3:ListAllMyBuckets",
"s3:ListBucket",
"s3:GetBucketLocation",
"iam:ListRoles",
"iam:GetRole",
],
))
# **********S3 Batch Operations Role******************************
s3BatchOperationsRole = iam.Role(
self,
"S3BatchOperationsRole",
assumed_by=iam.ServicePrincipal(
"batchoperations.s3.amazonaws.com"),
)
# **********S3 Bucket******************************
# S3 bucket for input documents and output
contentBucket = s3.Bucket(self, "DocumentsBucket", versioned=False)
existingContentBucket = s3.Bucket(self,
"ExistingDocumentsBucket",
versioned=False)
existingContentBucket.grant_read_write(s3BatchOperationsRole)
inventoryAndLogsBucket = s3.Bucket(self,
"InventoryAndLogsBucket",
versioned=False)
inventoryAndLogsBucket.grant_read_write(s3BatchOperationsRole)
# **********DynamoDB Table*************************
# DynamoDB table with links to output in S3
outputTable = dynamodb.Table(
self,
"OutputTable",
partition_key={
"name": "documentId",
"type": dynamodb.AttributeType.STRING,
},
sort_key={
"name": "outputType",
"type": dynamodb.AttributeType.STRING,
},
)
# DynamoDB table with links to output in S3
documentsTable = dynamodb.Table(
self,
"DocumentsTable",
partition_key={
"name": "documentId",
"type": dynamodb.AttributeType.STRING,
},
stream=dynamodb.StreamViewType.NEW_IMAGE,
)
# **********SQS Queues*****************************
# DLQ (Dead Letter Queue)
dlq = sqs.Queue(
self,
"DLQ",
visibility_timeout=core.Duration.seconds(30),
retention_period=core.Duration.seconds(1209600),
)
# Input Queue for sync jobs
syncJobsQueue = sqs.Queue(
self,
"SyncJobs",
visibility_timeout=core.Duration.seconds(30),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue={
"queue": dlq,
"max_receive_count": 50
},
)
# Input Queue for async jobs
asyncJobsQueue = sqs.Queue(
self,
"AsyncJobs",
visibility_timeout=core.Duration.seconds(30),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue={
"queue": dlq,
"max_receive_count": 50
},
)
# Queue
jobResultsQueue = sqs.Queue(
self,
"JobResults",
visibility_timeout=core.Duration.seconds(900),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue={
"queue": dlq,
"max_receive_count": 50
},
)
# Trigger
# jobCompletionTopic.subscribeQueue(jobResultsQueue)
jobCompletionTopic.add_subscription(
snsSubscriptions.SqsSubscription(jobResultsQueue))
# **********Lambda Functions******************************
# Helper Layer with helper functions
helperLayer = _lambda.LayerVersion(
self,
"HelperLayer",
code=_lambda.Code.from_asset("awscdk/lambda/helper"),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
license="Apache-2.0",
description="Helper layer.",
)
# Textractor helper layer
textractorLayer = _lambda.LayerVersion(
self,
"Textractor",
code=_lambda.Code.from_asset("awscdk/lambda/textractor"),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
license="Apache-2.0",
description="Textractor layer.",
)
# -----------------------------------------------------------
# S3 Event processor
s3Processor = _lambda.Function(
self,
"S3Processor",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset("awscdk/lambda/s3processor"),
handler="lambda_function.lambda_handler",
environment={
"SYNC_QUEUE_URL": syncJobsQueue.queue_url,
"ASYNC_QUEUE_URL": asyncJobsQueue.queue_url,
"DOCUMENTS_TABLE": documentsTable.table_name,
"OUTPUT_TABLE": outputTable.table_name,
},
)
# Layer
s3Processor.add_layers(helperLayer)
# Trigger
s3Processor.add_event_source(
S3EventSource(contentBucket, events=[s3.EventType.OBJECT_CREATED]))
# Permissions
documentsTable.grant_read_write_data(s3Processor)
syncJobsQueue.grant_send_messages(s3Processor)
asyncJobsQueue.grant_send_messages(s3Processor)
# ------------------------------------------------------------
# S3 Batch Operations Event processor
s3BatchProcessor = _lambda.Function(
self,
"S3BatchProcessor",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset("awscdk/lambda/s3batchprocessor"),
handler="lambda_function.lambda_handler",
environment={
"DOCUMENTS_TABLE": documentsTable.table_name,
"OUTPUT_TABLE": outputTable.table_name,
},
reserved_concurrent_executions=1,
)
# Layer
s3BatchProcessor.add_layers(helperLayer)
# Permissions
documentsTable.grant_read_write_data(s3BatchProcessor)
s3BatchProcessor.grant_invoke(s3BatchOperationsRole)
s3BatchOperationsRole.add_to_policy(
iam.PolicyStatement(actions=["lambda:*"], resources=["*"]))
# ------------------------------------------------------------
# Document processor (Router to Sync/Async Pipeline)
documentProcessor = _lambda.Function(
self,
"TaskProcessor",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset("awscdk/lambda/documentprocessor"),
handler="lambda_function.lambda_handler",
environment={
"SYNC_QUEUE_URL": syncJobsQueue.queue_url,
"ASYNC_QUEUE_URL": asyncJobsQueue.queue_url,
},
)
# Layer
documentProcessor.add_layers(helperLayer)
# Trigger
documentProcessor.add_event_source(
DynamoEventSource(
documentsTable,
starting_position=_lambda.StartingPosition.TRIM_HORIZON,
))
# Permissions
documentsTable.grant_read_write_data(documentProcessor)
syncJobsQueue.grant_send_messages(documentProcessor)
asyncJobsQueue.grant_send_messages(documentProcessor)
# ------------------------------------------------------------
# Sync Jobs Processor (Process jobs using sync APIs)
syncProcessor = _lambda.Function(
self,
"SyncProcessor",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset("awscdk/lambda/documentprocessor"),
handler="lambda_function.lambda_handler",
environment={
"OUTPUT_TABLE": outputTable.table_name,
"DOCUMENTS_TABLE": documentsTable.table_name,
"AWS_DATA_PATH": "models",
},
reserved_concurrent_executions=1,
timeout=core.Duration.seconds(25),
)
# Layer
syncProcessor.add_layers(helperLayer)
syncProcessor.add_layers(textractorLayer)
# Trigger
syncProcessor.add_event_source(
SqsEventSource(syncJobsQueue, batch_size=1))
# Permissions
contentBucket.grant_read_write(syncProcessor)
existingContentBucket.grant_read_write(syncProcessor)
outputTable.grant_read_write_data(syncProcessor)
documentsTable.grant_read_write_data(syncProcessor)
syncProcessor.add_to_role_policy(
iam.PolicyStatement(actions=["textract:*"], resources=["*"]))
# ------------------------------------------------------------
# Async Job Processor (Start jobs using Async APIs)
asyncProcessor = _lambda.Function(
self,
"ASyncProcessor",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset("awscdk/lambda/asyncprocessor"),
handler="lambda_function.lambda_handler",
environment={
"ASYNC_QUEUE_URL": asyncJobsQueue.queue_url,
"SNS_TOPIC_ARN": jobCompletionTopic.topic_arn,
"SNS_ROLE_ARN": textractServiceRole.role_arn,
"AWS_DATA_PATH": "models",
},
reserved_concurrent_executions=1,
timeout=core.Duration.seconds(60),
)
# asyncProcessor.addEnvironment("SNS_TOPIC_ARN", textractServiceRole.topic_arn)
# Layer
asyncProcessor.add_layers(helperLayer)
# Triggers
# Run async job processor every 5 minutes
# Enable code below after test deploy
rule = events.Rule(
self,
"Rule",
schedule=events.Schedule.expression("rate(2 minutes)"))
rule.add_target(LambdaFunction(asyncProcessor))
# Run when a job is successfully complete
asyncProcessor.add_event_source(SnsEventSource(jobCompletionTopic))
# Permissions
contentBucket.grant_read(asyncProcessor)
existingContentBucket.grant_read_write(asyncProcessor)
asyncJobsQueue.grant_consume_messages(asyncProcessor)
asyncProcessor.add_to_role_policy(
iam.PolicyStatement(
actions=["iam:PassRole"],
resources=[textractServiceRole.role_arn],
))
asyncProcessor.add_to_role_policy(
iam.PolicyStatement(actions=["textract:*"], resources=["*"]))
# ------------------------------------------------------------
# Async Jobs Results Processor
jobResultProcessor = _lambda.Function(
self,
"JobResultProcessor",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset("awscdk/lambda/jobresultprocessor"),
handler="lambda_function.lambda_handler",
memory_size=2000,
reserved_concurrent_executions=50,
timeout=core.Duration.seconds(900),
environment={
"OUTPUT_TABLE": outputTable.table_name,
"DOCUMENTS_TABLE": documentsTable.table_name,
"AWS_DATA_PATH": "models",
},
)
# Layer
jobResultProcessor.add_layers(helperLayer)
jobResultProcessor.add_layers(textractorLayer)
# Triggers
jobResultProcessor.add_event_source(
SqsEventSource(jobResultsQueue, batch_size=1))
# Permissions
outputTable.grant_read_write_data(jobResultProcessor)
documentsTable.grant_read_write_data(jobResultProcessor)
contentBucket.grant_read_write(jobResultProcessor)
existingContentBucket.grant_read_write(jobResultProcessor)
jobResultProcessor.add_to_role_policy(
iam.PolicyStatement(actions=["textract:*", "comprehend:*"],
resources=["*"]))
# --------------
# PDF Generator
pdfGenerator = _lambda.Function(
self,
"PdfGenerator",
runtime=_lambda.Runtime.JAVA_8,
code=_lambda.Code.from_asset("awscdk/lambda/pdfgenerator"),
handler="DemoLambdaV2::handleRequest",
memory_size=3000,
timeout=core.Duration.seconds(900),
)
contentBucket.grant_read_write(pdfGenerator)
existingContentBucket.grant_read_write(pdfGenerator)
pdfGenerator.grant_invoke(syncProcessor)
pdfGenerator.grant_invoke(asyncProcessor)
def create_event_handling(
self,
secrets: List[secretsmanager.Secret],
slack_host_ssm_name: str,
slack_webhook_ssm_name: str,
) -> lambda_.Function:
"""
Args:
secrets: a list of secrets that we will track for events
slack_host_ssm_name: the SSM parameter name for the slack host
slack_webhook_ssm_name: the SSM parameter name for the slack webhook id
Returns:
a lambda event handler
"""
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, "runtime/notify_slack")
env = {
# for the moment we don't parametrise at the CDK level.. only needed if this is liable to change
"SLACK_HOST_SSM_NAME": slack_host_ssm_name,
"SLACK_WEBHOOK_SSM_NAME": slack_webhook_ssm_name,
}
notifier = lambda_.Function(
self,
"NotifySlack",
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.AssetCode(filename),
handler="lambda_entrypoint.main",
timeout=Duration.minutes(1),
environment=env,
)
get_ssm_policy = PolicyStatement()
# there is some weirdness around SSM parameter ARN formation and leading slashes.. can't be bothered
# looking into right now - as the ones we want to use do a have a leading slash
# but put in this exception in case
if not slack_webhook_ssm_name.startswith(
"/") or not slack_host_ssm_name.startswith("/"):
raise Exception(
"SSM parameters need to start with a leading slash")
# see here - the *required* slash between parameter and the actual name uses the leading slash from the actual
# name itself.. which is wrong..
get_ssm_policy.add_resources(
f"arn:aws:ssm:*:*:parameter{slack_host_ssm_name}")
get_ssm_policy.add_resources(
f"arn:aws:ssm:*:*:parameter{slack_webhook_ssm_name}")
get_ssm_policy.add_actions("ssm:GetParameter")
notifier.add_to_role_policy(get_ssm_policy)
# we want a rule that traps all the rotation failures for our JWT secrets
rule = Rule(
self,
"NotifySlackRule",
)
rule.add_event_pattern(
source=["aws.secretsmanager"],
detail={
# at the moment only interested in these - add extra events into this array if wanting more
"eventName": ["RotationFailed", "RotationSucceeded"],
"additionalEventData": {
"SecretId": list(map(lambda s: s.secret_arn, secrets))
},
},
)
rule.add_target(LambdaFunction(notifier))
return notifier