def __init__(self, scope: core.Construct, id: str):
super().__init__(scope, id)
self.stack_name = core.Stack.of(self).stack_name
self.download_dlq = sqs.DeadLetterQueue(
max_receive_count=2,
queue=sqs.Queue(
self,
"DownloadDeadLetterQueue",
queue_name=f"{self.stack_name}-{names.DOWNLOAD_DLQ}",
retention_period=core.Duration.days(14),
),
)
self.download_queue = sqs.Queue(
self,
"DownloadQueue",
queue_name=f"{self.stack_name}-{names.DOWNLOAD_QUEUE}",
retention_period=core.Duration.days(14),
visibility_timeout=core.Duration.seconds(300),
dead_letter_queue=self.download_dlq,
)
self.upload_dlq = sqs.DeadLetterQueue(
max_receive_count=2,
queue=sqs.Queue(
self,
"UploadDeadLetterQueue",
queue_name=f"{self.stack_name}-{names.UPLOAD_DLQ}",
retention_period=core.Duration.days(14),
),
)
self.upload_queue = sqs.Queue(
self,
"UploadQueue",
queue_name=f"{self.stack_name}-{names.UPLOAD_QUEUE}",
retention_period=core.Duration.days(14),
visibility_timeout=core.Duration.seconds(300),
content_based_deduplication=True,
fifo=True,
dead_letter_queue=self.upload_dlq,
)
# note: you can't simply use the `queue_name` attribute of the
# construct objects here: cdk/cfn will complain about
# 'The Name field of Export must not depend on any resources'
queues = {
names.DOWNLOAD_QUEUE: self.download_queue,
names.DOWNLOAD_DLQ: self.download_dlq.queue,
names.UPLOAD_QUEUE: self.upload_queue,
names.UPLOAD_DLQ: self.upload_dlq.queue,
}
for name, queue in queues.items():
core.CfnOutput(
self,
f"{name}-queue-export",
export_name=f"{self.stack_name}-{name.replace('.', '-')}-url",
value=queue.queue_url,
)
def __init__(self, scope: core.Construct, _id: str, bucket_para, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
self.ddb_file_list = ddb.Table(self, "ddb",
table_name=table_queue_name,
partition_key=ddb.Attribute(name="Key", type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
self.sqs_queue_DLQ = sqs.Queue(self, "sqs_DLQ",
queue_name=table_queue_name + "-DLQ",
visibility_timeout=core.Duration.hours(1),
retention_period=core.Duration.days(14)
)
self.sqs_queue = sqs.Queue(self, "sqs_queue",
queue_name=table_queue_name,
visibility_timeout=core.Duration.hours(1),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=24,
queue=self.sqs_queue_DLQ
)
)
self.ssm_bucket_para = ssm.StringParameter(self, "para-bucket",
string_value=json.dumps(bucket_para),
parameter_name=ssm_parameter_bucket
)
# You need to manually setup ssm_credential_para in SSM Parameter Store before deploy CDK
# Here import ssm_credential_para, MIND THE VERSION NUMBER MUST BE EXACT THE SAME !!!
# 你需要先手工配置了一个ssm_credential_para，然后在这里导入，注意版本号一致!!!
self.ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes(
self, "ssm_parameter_credentials",
parameter_name=ssm_parameter_credentials,
version=2
)
def __init__(self, scope: core.Construct, construct_id: str,
queue_context: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
q = dict(self.node.try_get_context(queue_context))
queue_dlq = _sqs.Queue(self,
q["queue_dlq_name"],
queue_name=q["queue_dlq_name"])
queue = _sqs.Queue(
self,
q["queue_name"],
queue_name=q["queue_name"],
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=q["queue_dlq_max_receive_count"],
queue=queue_dlq),
encryption=_sqs.QueueEncryption.KMS_MANAGED,
visibility_timeout=Duration.seconds(30),
delivery_delay=Duration.seconds(15),
retention_period=Duration.hours(14),
)
self.queue = queue
self.queue_dlq = queue_dlq
# Outputs
core.CfnOutput(self, "QueueUrl", value=queue.queue_url)
def __init__(self, scope: core.Construct, _id: str, bucket_para,
**kwargs) -> None:
super().__init__(scope, _id, **kwargs)
self.ddb_file_list = ddb.Table(
self,
"s3_migrate_ddb",
partition_key=ddb.Attribute(name="Key",
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
self.sqs_queue_DLQ = sqs.Queue(
self,
"s3_migrate_sqs_DLQ",
visibility_timeout=core.Duration.hours(1),
retention_period=core.Duration.days(14))
self.sqs_queue = sqs.Queue(self,
"s3_migrate_sqs_queue",
visibility_timeout=core.Duration.hours(1),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=24,
queue=self.sqs_queue_DLQ))
self.ssm_bucket_para = ssm.StringParameter(self,
"s3_migrate_bucket_para",
string_value=json.dumps(
bucket_para, indent=4))
# You need to manually setup ssm_credential_para in SSM Parameter Store before deploy CDK
# Here import ssm_credential_para, MIND THE VERSION NUMBER MUST BE EXACT THE SAME !!!
# 你需要先手工配置了一个ssm_credential_para，然后在这里导入，注意版本号一致!!!
self.ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes(
self,
"ssm_parameter_credentials",
parameter_name=ssm_parameter_credentials,
version=1)
# New a S3 bucket, new object in this bucket will trigger SQS jobs
# This is not for existing S3 bucket. Jobsender will scan the existing bucket and create sqs jobs.
# 这里新建一个S3 bucket，里面新建Object就会触发SQS启动搬迁工作。
# 对于现有的S3 bucket，不在这里配置，由jobsender进行扫描并生成SQS Job任务。
self.s3bucket = s3.Bucket(self, "s3_migrate_bucket")
self.s3bucket.add_event_notification(
s3.EventType.OBJECT_CREATED, s3n.SqsDestination(self.sqs_queue))
# Deploy code
self.s3_deploy = s3.Bucket(self, "s3_migrate_deploybucket")
s3d.BucketDeployment(self,
"deploy_code",
sources=[s3d.Source.asset("./code")],
destination_bucket=self.s3_deploy)
core.CfnOutput(self,
'NewS3Bucket_MigrateObjects',
value=self.s3bucket.bucket_name)
core.CfnOutput(self,
'NewS3Bucket_deploy_code',
value=self.s3_deploy.bucket_name)
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
ddb_file_list = ddb.Table(self, "ddb",
partition_key=ddb.Attribute(name="Key", type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
sqs_queue_DLQ = sqs.Queue(self, "sqs_DLQ",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14)
)
sqs_queue = sqs.Queue(self, "sqs_queue",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=100,
queue=sqs_queue_DLQ
)
)
handler = lam.Function(self, "lambdaFunction",
code=lam.Code.asset("./lambda"),
handler="lambda_function.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
environment={
'table_name': ddb_file_list.table_name,
'queue_name': sqs_queue.queue_name,
'Des_bucket_default': Des_bucket_default,
'Des_prefix_default': Des_prefix_default,
'Des_region': Des_region,
'StorageClass': StorageClass,
'aws_access_key_id': aws_access_key_id,
'aws_secret_access_key': aws_secret_access_key
})
ddb_file_list.grant_read_write_data(handler)
handler.add_event_source(SqsEventSource(sqs_queue))
s3bucket = s3.Bucket(self, "s3bucket")
s3bucket.grant_read(handler)
s3bucket.add_event_notification(s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(sqs_queue))
# You can import an existing bucket and grant access to lambda
# exist_s3bucket = s3.Bucket.from_bucket_name(self, "import_bucket",
# bucket_name="you_bucket_name")
# exist_s3bucket.grant_read(handler)
# But You have to add sqs as imported bucket event notification manually, it doesn't support by CloudFormation
# An work around is to add on_cloud_trail_event for the bucket, but will trigger could_trail first
# 因为是导入的Bucket，需要手工建Bucket Event Trigger SQS，以及设置SQS允许该bucekt触发的Permission
core.CfnOutput(self, "DynamoDB_Table", value=ddb_file_list.table_name)
core.CfnOutput(self, "SQS_Job_Queue", value=sqs_queue.queue_name)
core.CfnOutput(self, "SQS_Job_Queue_DLQ", value=sqs_queue_DLQ.queue_name)
core.CfnOutput(self, "Worker_Lambda_Function", value=handler.function_name)
core.CfnOutput(self, "New_S3_Bucket", value=s3bucket.bucket_name)
def add_endpoint(self, bucket: s3.Bucket, fn: Function):
# create the queue
queue = sqs.Queue(self,
f'{fn.id_prefix}Queue',
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=5,
queue=sqs.Queue(
self,
f'{fn.id_prefix}DLQ',
queue_name=f'{fn.queue_name}-dlq')),
queue_name=fn.queue_name)
# create the receiver function
# add the queue url as an environment variable
receiver_function = lambda_.Function(
self,
f'{fn.id_prefix}ReceiverFunction',
code=fn.function_code,
environment={'QUEUE_URL': queue.queue_url},
function_name=f'{fn.function_name_prefix}-receiver',
handler=fn.receiver_function_handler,
layers=[fn.function_dependencies_layer],
# memory_size=256,
runtime=lambda_.Runtime.PYTHON_3_8)
# allow the receiver function to enqueue messages
queue.grant_send_messages(receiver_function)
# route requests to the receiver lambda
self.api.add_routes(integration=apigw.LambdaProxyIntegration(
handler=receiver_function),
methods=[fn.api_method],
path=fn.api_path)
# create the handler function
# add the bucket name as an environment variable
handler_function = lambda_.Function(
self,
f'{fn.id_prefix}HandlerFunction',
code=fn.function_code,
environment={'BUCKET_NAME': bucket.bucket_name},
function_name=f'{fn.function_name_prefix}-handler',
handler=fn.handler_function_handler,
layers=[fn.function_dependencies_layer],
# memory_size=256,
runtime=lambda_.Runtime.PYTHON_3_8)
# add the queue as a trigger for the handler function
handler_function.add_event_source(SqsEventSource(queue))
# allow the handler function to access the bucket
bucket.grant_read_write(handler_function)
def __init__(self, scope: core.Construct, construct_id: str, env,
**kwargs) -> None:
super().__init__(scope, construct_id, env=env, **kwargs)
# Create my-queue
my_dl_queue = sqs.Queue(self,
id="SQSTestDLQueueMovingMsg",
queue_name="my_queue_dl_test")
my_queue = sqs.Queue(self,
id="SQSTestMovingMsg",
queue_name="my_queue_test",
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=1, queue=my_dl_queue))
def __init__(self, scope: core.Construct, _id: str, bucket_para,
**kwargs) -> None:
super().__init__(scope, _id, **kwargs)
self.ddb_file_list = ddb.Table(
self,
"ddb",
table_name=table_queue_name,
partition_key=ddb.Attribute(name="Key",
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
self.sqs_queue_DLQ = sqs.Queue(
self,
"sqs_DLQ",
queue_name=table_queue_name + "-DLQ",
visibility_timeout=core.Duration.hours(1),
retention_period=core.Duration.days(14))
self.sqs_queue = sqs.Queue(self,
"sqs_queue",
queue_name=table_queue_name,
visibility_timeout=core.Duration.hours(1),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=24,
queue=self.sqs_queue_DLQ))
self.ssm_bucket_para = ssm.StringParameter(
self,
"para-bucket",
string_value=json.dumps(bucket_para),
parameter_name=ssm_parameter_bucket)
# 你需要先手工配置了一个ssm_credential_para，然后在这里导入，注意版本号一致!!!
self.ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes(
self,
"ssm_parameter_credentials",
parameter_name=ssm_parameter_credentials,
version=2)
# 这里新建一个S3 bucket，里面新建Object就会触发SQS启动搬迁工作。
# 对于现有的S3 bucket，不在这里配置，由jobsender进行扫描并生成SQS Job任务。
self.s3bucket = s3.Bucket(self, "newbucket")
self.s3bucket.add_event_notification(
s3.EventType.OBJECT_CREATED, s3n.SqsDestination(self.sqs_queue))
def __init__(self, scope: core.Construct, id: str, publisher_lambda,
suscriber_lambda, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
queue_fail = sqs.Queue(self,
"queue_fail",
visibility_timeout=core.Duration.seconds(20))
dlq = sqs.DeadLetterQueue(max_receive_count=100, queue=queue_fail)
self.queue = sqs.Queue(self,
"base_queue",
visibility_timeout=core.Duration.seconds(20),
dead_letter_queue=dlq)
#Configuramos la lambda para que reciba los mensajes de la cola
self.queue.grant_consume_messages(suscriber_lambda)
event_source = aws_lambda_event_sources.SqsEventSource(self.queue,
batch_size=1)
suscriber_lambda.add_event_source(event_source)
#la lambda que publica mensajes
self.queue.grant_send_messages(publisher_lambda)
def __init__(self, scope: core.Construct, id: str, stage: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.base_platform = BasePlatform(self, self.stack_name, stage)
queue = aws_sqs.Queue(self, "{}-queue".format(self.stack_name))
dlqueue = aws_sqs.DeadLetterQueue(max_receive_count=5, queue=queue)
# Create QueueProcessingEc2Service
ecs_service = aws_ecs_patterns.QueueProcessingEc2Service(
self,
"{}-svc".format(self.stack_name),
cluster=self.base_platform.ecs_cluster,
image=aws_ecs.ContainerImage.from_registry(
"amazon/amazon-ecs-sample"),
desired_task_count=0,
max_scaling_capacity=10,
memory_reservation_mib=1024,
cpu=512,
queue=queue,
)
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
# Setup SSM parameter of credentials, bucket_para, ignore_list
ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes(
self,
"ssm_parameter_credentials",
parameter_name=ssm_parameter_credentials,
version=1)
ssm_bucket_para = ssm.StringParameter(self,
"s3bucket_serverless",
string_value=json.dumps(
bucket_para, indent=4))
ssm_parameter_ignore_list = ssm.StringParameter(
self, "s3_migrate_ignore_list", string_value=ignore_list)
# Setup DynamoDB
ddb_file_list = ddb.Table(self,
"s3migrate_serverless",
partition_key=ddb.Attribute(
name="Key",
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
ddb_file_list.add_global_secondary_index(
partition_key=ddb.Attribute(name="desBucket",
type=ddb.AttributeType.STRING),
index_name="desBucket-index",
projection_type=ddb.ProjectionType.INCLUDE,
non_key_attributes=["desKey", "versionId"])
# Setup SQS
sqs_queue_DLQ = sqs.Queue(self,
"s3migrate_serverless_Q_DLQ",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14))
sqs_queue = sqs.Queue(self,
"s3migrate_serverless_Q",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=60, queue=sqs_queue_DLQ))
# Setup API for Lambda to get IP address (for debug networking routing purpose)
checkip = api.RestApi(
self,
"lambda-checkip-api",
cloud_watch_role=True,
deploy=True,
description="For Lambda get IP address",
default_integration=api.MockIntegration(
integration_responses=[
api.IntegrationResponse(status_code="200",
response_templates={
"application/json":
"$context.identity.sourceIp"
})
],
request_templates={"application/json": '{"statusCode": 200}'}),
endpoint_types=[api.EndpointType.REGIONAL])
checkip.root.add_method("GET",
method_responses=[
api.MethodResponse(
status_code="200",
response_models={
"application/json":
api.Model.EMPTY_MODEL
})
])
# Setup Lambda functions
handler = lam.Function(self,
"s3-migrate-worker",
code=lam.Code.asset("./lambda"),
handler="lambda_function_worker.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name':
ddb_file_list.table_name,
'Des_bucket_default': Des_bucket_default,
'Des_prefix_default': Des_prefix_default,
'StorageClass': StorageClass,
'checkip_url': checkip.url,
'ssm_parameter_credentials':
ssm_parameter_credentials,
'JobType': JobType,
'MaxRetry': MaxRetry,
'MaxThread': MaxThread,
'MaxParallelFile': MaxParallelFile,
'JobTimeout': JobTimeout,
'UpdateVersionId': UpdateVersionId,
'GetObjectWithVersionId':
GetObjectWithVersionId
})
handler_jobsender = lam.Function(
self,
"s3-migrate-jobsender",
code=lam.Code.asset("./lambda"),
handler="lambda_function_jobsender.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name': ddb_file_list.table_name,
'StorageClass': StorageClass,
'checkip_url': checkip.url,
'sqs_queue': sqs_queue.queue_name,
'ssm_parameter_credentials': ssm_parameter_credentials,
'ssm_parameter_ignore_list':
ssm_parameter_ignore_list.parameter_name,
'ssm_parameter_bucket': ssm_bucket_para.parameter_name,
'JobType': JobType,
'MaxRetry': MaxRetry,
'JobsenderCompareVersionId': JobsenderCompareVersionId
})
# Allow lambda read/write DDB, SQS
ddb_file_list.grant_read_write_data(handler)
ddb_file_list.grant_read_write_data(handler_jobsender)
sqs_queue.grant_send_messages(handler_jobsender)
# SQS trigger Lambda worker
handler.add_event_source(SqsEventSource(sqs_queue, batch_size=1))
# Option1: Create S3 Bucket, all new objects in this bucket will be transmitted by Lambda Worker
s3bucket = s3.Bucket(self, "s3_new_migrate")
s3bucket.grant_read(handler)
s3bucket.add_event_notification(s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(sqs_queue))
# Option2: Allow Exist S3 Buckets to be read by Lambda functions.
# Lambda Jobsender will scan and compare the these buckets and trigger Lambda Workers to transmit
bucket_name = ''
for b in bucket_para:
if bucket_name != b['src_bucket']: # 如果列了多个相同的Bucket，就跳过
bucket_name = b['src_bucket']
s3exist_bucket = s3.Bucket.from_bucket_name(
self,
bucket_name, # 用这个做id
bucket_name=bucket_name)
if JobType == 'PUT':
s3exist_bucket.grant_read(handler_jobsender)
s3exist_bucket.grant_read(handler)
else: # 'GET' mode
s3exist_bucket.grant_read_write(handler_jobsender)
s3exist_bucket.grant_read_write(handler)
# Allow Lambda read ssm parameters
ssm_bucket_para.grant_read(handler_jobsender)
ssm_credential_para.grant_read(handler)
ssm_credential_para.grant_read(handler_jobsender)
ssm_parameter_ignore_list.grant_read(handler_jobsender)
# Schedule cron event to trigger Lambda Jobsender per hour:
event.Rule(self,
'cron_trigger_jobsender',
schedule=event.Schedule.rate(core.Duration.hours(1)),
targets=[target.LambdaFunction(handler_jobsender)])
# TODO: Trigger event imediately, add custom resource lambda to invoke handler_jobsender
# Create Lambda logs filter to create network traffic metric
handler.log_group.add_metric_filter(
"Completed-bytes",
metric_name="Completed-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Complete", bytes, key]'))
handler.log_group.add_metric_filter(
"Uploading-bytes",
metric_name="Uploading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Uploading", bytes, key]'))
handler.log_group.add_metric_filter(
"Downloading-bytes",
metric_name="Downloading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Downloading", bytes, key]'))
handler.log_group.add_metric_filter(
"MaxMemoryUsed",
metric_name="MaxMemoryUsed",
metric_namespace="s3_migrate",
metric_value="$memory",
filter_pattern=logs.FilterPattern.literal(
'[head="REPORT", a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, '
'a13, a14, a15, a16, memory, MB="MB", rest]'))
lambda_metric_Complete = cw.Metric(namespace="s3_migrate",
metric_name="Completed-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Upload = cw.Metric(namespace="s3_migrate",
metric_name="Uploading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Download = cw.Metric(namespace="s3_migrate",
metric_name="Downloading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_MaxMemoryUsed = cw.Metric(
namespace="s3_migrate",
metric_name="MaxMemoryUsed",
statistic="Maximum",
period=core.Duration.minutes(1))
handler.log_group.add_metric_filter(
"ERROR",
metric_name="ERROR-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"ERROR"'))
handler.log_group.add_metric_filter(
"WARNING",
metric_name="WARNING-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"WARNING"'))
# Task timed out
handler.log_group.add_metric_filter(
"TIMEOUT",
metric_name="TIMEOUT-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"Task timed out"'))
log_metric_ERROR = cw.Metric(namespace="s3_migrate",
metric_name="ERROR-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
log_metric_WARNING = cw.Metric(namespace="s3_migrate",
metric_name="WARNING-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
log_metric_TIMEOUT = cw.Metric(namespace="s3_migrate",
metric_name="TIMEOUT-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
# Dashboard to monitor SQS and Lambda
board = cw.Dashboard(self, "s3_migrate_serverless")
board.add_widgets(
cw.GraphWidget(title="Lambda-NETWORK",
left=[
lambda_metric_Download, lambda_metric_Upload,
lambda_metric_Complete
]),
cw.GraphWidget(title="Lambda-concurrent",
left=[
handler.metric(
metric_name="ConcurrentExecutions",
period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="Lambda-invocations/errors/throttles",
left=[
handler.metric_invocations(
period=core.Duration.minutes(1)),
handler.metric_errors(period=core.Duration.minutes(1)),
handler.metric_throttles(period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="Lambda-duration",
left=[
handler.metric_duration(period=core.Duration.minutes(1))
]),
)
board.add_widgets(
cw.GraphWidget(title="Lambda_MaxMemoryUsed(MB)",
left=[lambda_metric_MaxMemoryUsed]),
cw.GraphWidget(title="ERROR/WARNING Logs",
left=[log_metric_ERROR],
right=[log_metric_WARNING, log_metric_TIMEOUT]),
cw.GraphWidget(
title="SQS-Jobs",
left=[
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1))
]),
cw.SingleValueWidget(
title="Running/Waiting and Dead Jobs",
metrics=[
sqs_queue.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)),
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1))
],
height=6))
# Alarm for queue - DLQ
alarm_DLQ = cw.Alarm(
self,
"SQS_DLQ",
metric=sqs_queue_DLQ.metric_approximate_number_of_messages_visible(
),
threshold=0,
comparison_operator=cw.ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluation_periods=1,
datapoints_to_alarm=1)
alarm_topic = sns.Topic(self, "SQS queue-DLQ has dead letter")
alarm_topic.add_subscription(
subscription=sub.EmailSubscription(alarm_email))
alarm_DLQ.add_alarm_action(action.SnsAction(alarm_topic))
core.CfnOutput(self,
"Dashboard",
value="CloudWatch Dashboard name s3_migrate_serverless")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
###########################################################################
# AWS SECRETS MANAGER - Templated secret
###########################################################################
# templated_secret = aws_secretsmanager.Secret(self, "TemplatedSecret",
# generate_secret_string=aws_secretsmanager.SecretStringGenerator(
# secret_string_template= "{\"username\":\"cleanbox\"}",
# generate_string_key="password"
# )
# )
###########################################################################
# CUSTOM CLOUDFORMATION RESOURCE
###########################################################################
# customlambda = aws_lambda.Function(self,'customconfig',
# handler='customconfig.on_event',
# runtime=aws_lambda.Runtime.PYTHON_3_7,
# code=aws_lambda.Code.asset('customconfig'),
# )
# customlambda_statement = aws_iam.PolicyStatement(actions=["events:PutRule"], conditions=None, effect=None, not_actions=None, not_principals=None, not_resources=None, principals=None, resources=["*"], sid=None)
# customlambda.add_to_role_policy(statement=customlambda_statement)
# my_provider = cr.Provider(self, "MyProvider",
# on_event_handler=customlambda,
# # is_complete_handler=is_complete, # optional async "waiter"
# log_retention=logs.RetentionDays.SIX_MONTHS
# )
# CustomResource(self, 'customconfigresource', service_token=my_provider.service_token)
###########################################################################
# AWS LAMBDA FUNCTIONS
###########################################################################
sqs_to_elastic_cloud = aws_lambda.Function(
self,
'sqs_to_elastic_cloud',
handler='sqs_to_elastic_cloud.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('sqs_to_elastic_cloud'),
memory_size=4096,
timeout=core.Duration.seconds(300),
log_retention=logs.RetentionDays.ONE_DAY)
sqs_to_elasticsearch_service = aws_lambda.Function(
self,
'sqs_to_elasticsearch_service',
handler='sqs_to_elasticsearch_service.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('sqs_to_elasticsearch_service'),
memory_size=4096,
timeout=core.Duration.seconds(300),
log_retention=logs.RetentionDays.ONE_DAY)
# sqs_to_elasticsearch_service.add_environment("kinesis_firehose_name", "-")
# sqs_to_elastic_cloud.add_environment("index_name", "-")
###########################################################################
# AWS LAMBDA FUNCTIONS
###########################################################################
# sqs_to_elasticsearch_service_permission = aws_lambda.Permission(*, principal, action=None, event_source_token=None, scope=None, source_account=None, source_arn=None)
###########################################################################
# AMAZON S3 BUCKETS
###########################################################################
access_log_bucket = aws_s3.Bucket(self, "access_log_bucket")
kinesis_log_bucket = aws_s3.Bucket(self, "kinesis_log_bucket")
###########################################################################
# LAMBDA SUPPLEMENTAL POLICIES
###########################################################################
lambda_supplemental_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=["s3:Get*", "s3:Head*", "s3:List*", "firehose:*"],
resources=["*"])
sqs_to_elastic_cloud.add_to_role_policy(
lambda_supplemental_policy_statement)
sqs_to_elasticsearch_service.add_to_role_policy(
lambda_supplemental_policy_statement)
###########################################################################
# AWS SNS TOPICS
###########################################################################
access_log_topic = aws_sns.Topic(self, "access_log_topic")
###########################################################################
# ADD AMAZON S3 BUCKET NOTIFICATIONS
###########################################################################
access_log_bucket.add_event_notification(
aws_s3.EventType.OBJECT_CREATED,
aws_s3_notifications.SnsDestination(access_log_topic))
###########################################################################
# AWS SQS QUEUES
###########################################################################
sqs_to_elasticsearch_service_queue_iqueue = aws_sqs.Queue(
self, "sqs_to_elasticsearch_service_queue_dlq")
sqs_to_elasticsearch_service_queue_dlq = aws_sqs.DeadLetterQueue(
max_receive_count=10,
queue=sqs_to_elasticsearch_service_queue_iqueue)
sqs_to_elasticsearch_service_queue = aws_sqs.Queue(
self,
"sqs_to_elasticsearch_service_queue",
visibility_timeout=core.Duration.seconds(301),
dead_letter_queue=sqs_to_elasticsearch_service_queue_dlq)
sqs_to_elastic_cloud_queue_iqueue = aws_sqs.Queue(
self, "sqs_to_elastic_cloud_queue_dlq")
sqs_to_elastic_cloud_queue_dlq = aws_sqs.DeadLetterQueue(
max_receive_count=10, queue=sqs_to_elastic_cloud_queue_iqueue)
sqs_to_elastic_cloud_queue = aws_sqs.Queue(
self,
"sqs_to_elastic_cloud_queue",
visibility_timeout=core.Duration.seconds(301),
dead_letter_queue=sqs_to_elastic_cloud_queue_dlq)
###########################################################################
# AWS SNS TOPIC SUBSCRIPTIONS
###########################################################################
access_log_topic.add_subscription(
aws_sns_subscriptions.SqsSubscription(sqs_to_elastic_cloud_queue))
access_log_topic.add_subscription(
aws_sns_subscriptions.SqsSubscription(
sqs_to_elasticsearch_service_queue))
###########################################################################
# AWS LAMBDA SQS EVENT SOURCE
###########################################################################
sqs_to_elastic_cloud.add_event_source(
SqsEventSource(sqs_to_elastic_cloud_queue, batch_size=10))
sqs_to_elasticsearch_service.add_event_source(
SqsEventSource(sqs_to_elasticsearch_service_queue, batch_size=10))
###########################################################################
# AWS ELASTICSEARCH DOMAIN
###########################################################################
###########################################################################
# AWS ELASTICSEARCH DOMAIN ACCESS POLICY
###########################################################################
this_aws_account = aws_iam.AccountPrincipal(account_id="012345678912")
# s3_to_elasticsearch_access_logs_domain_access_policy_statement = aws_iam.PolicyStatement(
# principals=[this_aws_account],
# effect=aws_iam.Effect.ALLOW,
# actions=["es:*"],
# resources=["*"]
# )
# s3_to_elasticsearch_access_logs_domain_access_policy_statement_list=[]
# s3_to_elasticsearch_access_logs_domain_access_policy_statement_list.append(s3_to_elasticsearch_access_logs_domain_access_policy_statement)
s3_to_elasticsearch_access_logs_domain = aws_elasticsearch.Domain(
self,
"s3-to-elasticsearch-access-logs-domain",
# access_policies=s3_to_elasticsearch_access_logs_domain_access_policy_statement_list,
version=aws_elasticsearch.ElasticsearchVersion.V7_1,
capacity={
"master_nodes": 3,
"data_nodes": 4
},
ebs={"volume_size": 100},
zone_awareness={"availability_zone_count": 2},
logging={
"slow_search_log_enabled": True,
"app_log_enabled": True,
"slow_index_log_enabled": True
})
###########################################################################
# AMAZON COGNITO USER POOL
###########################################################################
s3_to_elasticsearch_user_pool = aws_cognito.UserPool(
self,
"s3-to-elasticsearch-access-logs-pool",
account_recovery=None,
auto_verify=None,
custom_attributes=None,
email_settings=None,
enable_sms_role=None,
lambda_triggers=None,
mfa=None,
mfa_second_factor=None,
password_policy=None,
self_sign_up_enabled=None,
sign_in_aliases=aws_cognito.SignInAliases(email=True,
phone=None,
preferred_username=None,
username=True),
sign_in_case_sensitive=None,
sms_role=None,
sms_role_external_id=None,
standard_attributes=None,
user_invitation=None,
user_pool_name=None,
user_verification=None)
###########################################################################
# AMAZON KINESIS FIREHOSE STREAM
###########################################################################
# kinesis_policy_statement = aws_iam.PolicyStatement(
# effect=aws_iam.Effect.ALLOW,
# # actions=["es:*", "s3:*", "kms:*", "kinesis:*", "lambda:*"],
# actions=["*"],
# resources=["*"]
# )
# kinesis_policy_document = aws_iam.PolicyDocument()
# kinesis_policy_document.add_statements(kinesis_policy_statement)
kinesis_firehose_stream_role = aws_iam.Role(
self,
"BaseVPCIAMLogRole",
assumed_by=aws_iam.ServicePrincipal('firehose.amazonaws.com'),
role_name=None,
inline_policies={
"AllowLogAccess":
aws_iam.PolicyDocument(
assign_sids=False,
statements=[
aws_iam.PolicyStatement(actions=[
'*', 'es:*', 'logs:PutLogEvents',
'logs:DescribeLogGroups',
'logs:DescribeLogsStreams'
],
effect=aws_iam.Effect('ALLOW'),
resources=['*'])
])
})
RetryOptions = aws_kinesisfirehose.CfnDeliveryStream.ElasticsearchRetryOptionsProperty(
duration_in_seconds=300)
s3_configuration = aws_kinesisfirehose.CfnDeliveryStream.S3DestinationConfigurationProperty(
bucket_arn=kinesis_log_bucket.bucket_arn,
role_arn=kinesis_firehose_stream_role.role_arn)
ElasticsearchDestinationConfiguration = aws_kinesisfirehose.CfnDeliveryStream.ElasticsearchDestinationConfigurationProperty(
# "BufferingHints" : ElasticsearchBufferingHints,
# "CloudWatchLoggingOptions" : CloudWatchLoggingOptions,
# "ClusterEndpoint" : String,
domain_arn=s3_to_elasticsearch_access_logs_domain.domain_arn,
index_name="s3-to-elasticsearch-accesslogs",
index_rotation_period="OneDay",
# "ProcessingConfiguration" : ProcessingConfiguration,
retry_options=RetryOptions,
role_arn=kinesis_firehose_stream_role.role_arn,
# "S3BackupMode" : String,
s3_configuration=s3_configuration
# "TypeName" : String
# "VpcConfiguration" : VpcConfiguration
)
kinesis_firehose_stream = aws_kinesisfirehose.CfnDeliveryStream(
self,
"kinesis_firehose_stream",
delivery_stream_encryption_configuration_input=None,
delivery_stream_name=None,
delivery_stream_type=None,
elasticsearch_destination_configuration=
ElasticsearchDestinationConfiguration,
extended_s3_destination_configuration=None,
http_endpoint_destination_configuration=None,
kinesis_stream_source_configuration=None,
redshift_destination_configuration=None,
s3_destination_configuration=None,
splunk_destination_configuration=None,
tags=None)
sqs_to_elasticsearch_service.add_environment(
"FIREHOSE_NAME", kinesis_firehose_stream.ref)
sqs_to_elasticsearch_service.add_environment(
"QUEUEURL", sqs_to_elasticsearch_service_queue.queue_url)
sqs_to_elasticsearch_service.add_environment("DEBUG", "False")
sqs_to_elastic_cloud.add_environment("ELASTICCLOUD_SECRET_NAME", "-")
sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_ID", "-")
sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_PASSWORD", "-")
sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_USERNAME", "-")
sqs_to_elastic_cloud.add_environment(
"QUEUEURL", sqs_to_elastic_cloud_queue.queue_url)
sqs_to_elastic_cloud.add_environment("DEBUG", "False")
def __init__(self, app: core.App, id: str) -> None:
super().__init__(app, id)
##################################
# Resource Property Config
##################################
# see: https://docs.aws.amazon.com/lambda/latest/dg/
# with-sqs.html#events-sqs-queueconfig
domain_scan_timeout = 900
queue_visibility_timeout = 6 * domain_scan_timeout
# tldextract needs to cache tld list after request and /tmp is writable in Lamdba
tld_cache = os.path.join('/tmp', '.tld_set')
##################################
# Domain Gatherer Lambda and Queue
##################################
# create queue
domain_queue = sqs.Queue(
self,
'domain-queue',
visibility_timeout=core.Duration.seconds(queue_visibility_timeout),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=5,
queue=sqs.Queue(self,
'domain-queue-dlq',
retention_period=core.Duration.days(5))))
# create lambda to gather domains
domain_gatherer_lambda = lambda_.Function(
self,
"domain-gatherer",
code=lambda_.Code.from_asset(
'lambda-releases/domain-gatherer.zip'),
handler="handler.main",
timeout=core.Duration.seconds(600),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=400)
# set env vars
domain_gatherer_lambda.add_environment('SQS_URL',
domain_queue.queue_url)
domain_gatherer_lambda.add_environment('TLDEXTRACT_CACHE', tld_cache)
# provide lambda with execution role
domain_gatherer_lambda_exec_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'lambda:InvokeFunction', 'sqs:SendMessage',
'sqs:DeleteMessage', 'sqs:SendMessageBatch',
'sqs:SetQueueAttributes', "sqs:GetQueueAttributes",
"sqs:GetQueueUrl", 'sqs:GetQueueAttributes'
],
resources=[domain_queue.queue_arn])
domain_gatherer_lambda.add_to_role_policy(
domain_gatherer_lambda_exec_policy)
# allow lambda to send messages to queue
domain_queue.grant_send_messages(domain_gatherer_lambda)
# create rule to run the lambda every Friday
rule = events.Rule(
self,
"Rule",
schedule=events.Schedule.cron(minute='0',
hour='18',
month='*',
week_day='FRI',
year='*'),
)
rule.add_target(targets.LambdaFunction(domain_gatherer_lambda))
##################################
# Domain Scan Lambda and Results Bucket
##################################
# create lambda to scan domains
domain_scanner_lambda = lambda_.Function(
self,
"domain-scanner",
code=lambda_.Code.from_asset('lambda-releases/domain-scanner.zip'),
handler="handler.main",
timeout=core.Duration.seconds(domain_scan_timeout),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=1000)
# create sqs event source for domain scan lambda
domain_scanner_lambda.add_event_source(
sources.SqsEventSource(domain_queue, batch_size=2))
# create s3 bucket to put results
bucket = s3.Bucket(self,
'results-bucket',
versioned=False,
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess(
block_public_acls=False,
ignore_public_acls=False,
block_public_policy=True,
restrict_public_buckets=True))
# grant s3:PUT to the pa11y lambda
bucket.grant_put(domain_scanner_lambda)
# set an env var for bucket name
domain_scanner_lambda.add_environment('BUCKET_NAME',
bucket.bucket_name)
# create execution role for domain scanner lambda
domain_scanner_lambda_exec_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'lambda:InvokeFunction', 'sqs:SendMessage',
'sqs:DeleteMessage', 'sqs:SendMessageBatch',
'sqs:SetQueueAttributes', "sqs:GetQueueAttributes",
"sqs:GetQueueUrl", 'sqs:GetQueueAttributes', "s3:PutObject",
"s3:PutObjectAcl"
],
resources=[bucket.bucket_arn])
domain_scanner_lambda.add_to_role_policy(
domain_scanner_lambda_exec_policy)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
###########################################################################
# AWS SECRETS MANAGER - Templated secret
###########################################################################
# templated_secret = aws_secretsmanager.Secret(self, "TemplatedSecret",
# generate_secret_string=aws_secretsmanager.SecretStringGenerator(
# secret_string_template= "{\"username\":\"cleanbox\"}",
# generate_string_key="password"
# )
# )
###########################################################################
# CUSTOM CLOUDFORMATION RESOURCE
###########################################################################
# customlambda = aws_lambda.Function(self,'customconfig',
# handler='customconfig.on_event',
# runtime=aws_lambda.Runtime.PYTHON_3_7,
# code=aws_lambda.Code.asset('customconfig'),
# )
# customlambda_statement = aws_iam.PolicyStatement(actions=["events:PutRule"], conditions=None, effect=None, not_actions=None, not_principals=None, not_resources=None, principals=None, resources=["*"], sid=None)
# customlambda.add_to_role_policy(statement=customlambda_statement)
# my_provider = cr.Provider(self, "MyProvider",
# on_event_handler=customlambda,
# # is_complete_handler=is_complete, # optional async "waiter"
# log_retention=logs.RetentionDays.SIX_MONTHS
# )
# CustomResource(self, 'customconfigresource', service_token=my_provider.service_token)
###########################################################################
# AWS LAMBDA FUNCTIONS
###########################################################################
sqs_to_elastic_cloud = aws_lambda.Function(
self,
'sqs_to_elastic_cloud',
handler='sqs_to_elastic_cloud.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('sqs_to_elastic_cloud'),
memory_size=4096,
timeout=core.Duration.seconds(301),
log_retention=logs.RetentionDays.ONE_DAY)
sqs_to_elasticsearch_service = aws_lambda.Function(
self,
'sqs_to_elasticsearch_service',
handler='sqs_to_elasticsearch_service.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('sqs_to_elasticsearch_service'),
memory_size=4096,
timeout=core.Duration.seconds(301),
log_retention=logs.RetentionDays.ONE_DAY)
###########################################################################
# AWS LAMBDA FUNCTIONS
###########################################################################
###########################################################################
# AMAZON S3 BUCKETS
###########################################################################
cloudtrail_log_bucket = aws_s3.Bucket(self, "cloudtrail_log_bucket")
###########################################################################
# LAMBDA SUPPLEMENTAL POLICIES
###########################################################################
lambda_supplemental_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=["s3:Get*", "s3:Head*", "s3:List*", "firehose:*", "es:*"],
resources=["*"])
sqs_to_elastic_cloud.add_to_role_policy(
lambda_supplemental_policy_statement)
sqs_to_elasticsearch_service.add_to_role_policy(
lambda_supplemental_policy_statement)
###########################################################################
# AWS SNS TOPICS
###########################################################################
cloudtrail_log_topic = aws_sns.Topic(self, "cloudtrail_log_topic")
###########################################################################
# ADD AMAZON S3 BUCKET NOTIFICATIONS
###########################################################################
cloudtrail_log_bucket.add_event_notification(
aws_s3.EventType.OBJECT_CREATED,
aws_s3_notifications.SnsDestination(cloudtrail_log_topic))
###########################################################################
# AWS SQS QUEUES
###########################################################################
sqs_to_elasticsearch_service_queue_iqueue = aws_sqs.Queue(
self, "sqs_to_elasticsearch_service_queue_dlq")
sqs_to_elasticsearch_service_queue_dlq = aws_sqs.DeadLetterQueue(
max_receive_count=10,
queue=sqs_to_elasticsearch_service_queue_iqueue)
sqs_to_elasticsearch_service_queue = aws_sqs.Queue(
self,
"sqs_to_elasticsearch_service_queue",
visibility_timeout=core.Duration.seconds(300),
dead_letter_queue=sqs_to_elasticsearch_service_queue_dlq)
sqs_to_elastic_cloud_queue_iqueue = aws_sqs.Queue(
self, "sqs_to_elastic_cloud_queue_dlq")
sqs_to_elastic_cloud_queue_dlq = aws_sqs.DeadLetterQueue(
max_receive_count=10, queue=sqs_to_elastic_cloud_queue_iqueue)
sqs_to_elastic_cloud_queue = aws_sqs.Queue(
self,
"sqs_to_elastic_cloud_queue",
visibility_timeout=core.Duration.seconds(300),
dead_letter_queue=sqs_to_elastic_cloud_queue_dlq)
###########################################################################
# AWS SNS TOPIC SUBSCRIPTIONS
###########################################################################
cloudtrail_log_topic.add_subscription(
aws_sns_subscriptions.SqsSubscription(sqs_to_elastic_cloud_queue))
cloudtrail_log_topic.add_subscription(
aws_sns_subscriptions.SqsSubscription(
sqs_to_elasticsearch_service_queue))
###########################################################################
# AWS LAMBDA SQS EVENT SOURCE
###########################################################################
sqs_to_elastic_cloud.add_event_source(
SqsEventSource(sqs_to_elastic_cloud_queue, batch_size=10))
sqs_to_elasticsearch_service.add_event_source(
SqsEventSource(sqs_to_elasticsearch_service_queue, batch_size=10))
###########################################################################
# AWS ELASTICSEARCH DOMAIN
###########################################################################
###########################################################################
# AWS ELASTICSEARCH DOMAIN ACCESS POLICY
###########################################################################
this_aws_account = aws_iam.AccountPrincipal(account_id="012345678912")
s3_to_elasticsearch_cloudtrail_logs_domain = aws_elasticsearch.Domain(
self,
"s3-to-elasticsearch-cloudtrail-logs-domain",
version=aws_elasticsearch.ElasticsearchVersion.V7_1,
capacity={
"master_nodes": 3,
"data_nodes": 4
},
ebs={"volume_size": 100},
zone_awareness={"availability_zone_count": 2},
logging={
"slow_search_log_enabled": True,
"app_log_enabled": True,
"slow_index_log_enabled": True
})
###########################################################################
# AMAZON COGNITO USER POOL
###########################################################################
s3_to_elasticsearch_user_pool = aws_cognito.UserPool(
self,
"s3-to-elasticsearch-cloudtrial-logs-pool",
account_recovery=None,
auto_verify=None,
custom_attributes=None,
email_settings=None,
enable_sms_role=None,
lambda_triggers=None,
mfa=None,
mfa_second_factor=None,
password_policy=None,
self_sign_up_enabled=None,
sign_in_aliases=aws_cognito.SignInAliases(email=True,
phone=None,
preferred_username=None,
username=True),
sign_in_case_sensitive=None,
sms_role=None,
sms_role_external_id=None,
standard_attributes=None,
user_invitation=None,
user_pool_name=None,
user_verification=None)
sqs_to_elasticsearch_service.add_environment(
"ELASTICSEARCH_HOST",
s3_to_elasticsearch_cloudtrail_logs_domain.domain_endpoint)
sqs_to_elasticsearch_service.add_environment(
"QUEUEURL", sqs_to_elasticsearch_service_queue.queue_url)
sqs_to_elasticsearch_service.add_environment("DEBUG", "False")
sqs_to_elastic_cloud.add_environment("ELASTICCLOUD_SECRET_NAME", "-")
sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_ID", "-")
sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_PASSWORD", "-")
sqs_to_elastic_cloud.add_environment("ELASTIC_CLOUD_USERNAME", "-")
sqs_to_elastic_cloud.add_environment(
"QUEUEURL", sqs_to_elastic_cloud_queue.queue_url)
sqs_to_elastic_cloud.add_environment("DEBUG", "False")
###########################################################################
# AWS COGNITO USER POOL
###########################################################################
allevents_trail = aws_cloudtrail.Trail(
self,
"allevents_trail",
bucket=cloudtrail_log_bucket,
cloud_watch_log_group=None,
cloud_watch_logs_retention=None,
enable_file_validation=None,
encryption_key=None,
include_global_service_events=None,
is_multi_region_trail=True,
kms_key=None,
management_events=aws_cloudtrail.ReadWriteType("ALL"),
s3_key_prefix=None,
send_to_cloud_watch_logs=False,
sns_topic=None,
trail_name=None)
def __init__(self, app: core.App, id: str) -> None:
super().__init__(app, id)
##################################
# Lambda Timeouts (seconds) & Queue Redrive
##################################
lambda_gatherer_timeout = 600
lambda_joiner_timeout = 350
# pa11y's timeout is set to 50, so the lambda is just a little longer
lambda_a11y_scan_timeout = 55
max_receive_count = 2
##################################
# S3 Bucket with Domains
##################################
asset = aws_s3_assets.Asset(
self, 'domain-list', path=os.path.abspath('./domains/domains.csv'))
##################################
# Domain Gatherer Lambda and Queue
##################################
domain_queue = sqs.Queue(
self,
'domain-queue',
visibility_timeout=core.Duration.seconds(
(max_receive_count + 1) * lambda_gatherer_timeout),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=max_receive_count,
queue=sqs.Queue(self,
'domain-queue-dlq',
retention_period=core.Duration.days(5))))
lambda_gatherer = lambda_.Function(
self,
"domain-gatherer",
code=lambda_.Code.from_asset('./lambdas/domain_gatherer'),
handler="handler.main",
timeout=core.Duration.seconds(lambda_gatherer_timeout),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=150)
lambda_gatherer.add_environment('SQS_URL', domain_queue.queue_url)
lambda_gatherer.add_environment('BUCKET_NAME', asset.s3_bucket_name)
lambda_gatherer.add_environment('OBJECT_KEY', asset.s3_object_key)
lambda_gatherer_sqs_exec_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'lambda:InvokeFunction', 'sqs:SendMessage',
'sqs:DeleteMessage', 'sqs:SendMessageBatch',
'sqs:SetQueueAttributes', 'sqs:GetQueueAttributes',
'sqs:GetQueueUrl', 'sqs:GetQueueAttributes'
],
resources=[domain_queue.queue_arn])
lambda_gatherer.add_to_role_policy(lambda_gatherer_sqs_exec_policy)
domain_queue.grant_send_messages(lambda_gatherer)
# trigger for 1st and 15th of the month at 18:00 UTC (1pm EST)
lambda_gatherer_rule = events.Rule(self,
"Lambda Gatherer Rule",
schedule=events.Schedule.cron(
minute='0',
hour='18',
day="1,15",
month='*',
year='*'))
lambda_gatherer_rule.add_target(
targets.LambdaFunction(lambda_gatherer))
asset.grant_read(lambda_gatherer)
##################################
# A11y Scanner Lambda and S3
##################################
layer = lambda_.LayerVersion(
self,
'chrome-aws-lambda',
code=lambda_.Code.from_asset('./lambdas/chrome_aws_lambda.zip'),
compatible_runtimes=[lambda_.Runtime.NODEJS_12_X],
description='A layer of chrome-aws-lambda')
lambda_a11y_scan = lambda_.Function(
self,
"a11y-scan",
code=lambda_.Code.from_asset('./lambdas/a11y_scan'),
handler="index.handler",
timeout=core.Duration.seconds(lambda_a11y_scan_timeout),
runtime=lambda_.Runtime.NODEJS_12_X,
memory_size=1000,
layers=[layer])
lambda_a11y_scan.add_event_source(
sources.SqsEventSource(domain_queue, batch_size=1))
# create s3 bucket to put results
results_bucket = s3.Bucket(self,
'results-bucket',
versioned=False,
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True),
lifecycle_rules=[
s3.LifecycleRule(
enabled=True,
expiration=core.Duration.days(10))
])
lambda_a11y_scan.add_environment('BUCKET_NAME',
results_bucket.bucket_name)
results_bucket.grant_put(lambda_a11y_scan)
##################################
# Results Joiner Lambda
##################################
# create s3 bucket to put site data
data_bucket = s3.Bucket(self,
'data-bucket',
versioned=False,
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True))
lambda_joiner = lambda_.Function(
self,
"results-joiner",
code=lambda_.Code.from_asset(
'./lambda-releases/results_joiner.zip'),
handler="handler.main",
timeout=core.Duration.seconds(lambda_joiner_timeout),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=400)
lambda_joiner.add_environment('DATA_BUCKET_NAME',
data_bucket.bucket_name)
lambda_joiner.add_environment('RESULTS_BUCKET_NAME',
results_bucket.bucket_name)
results_bucket.grant_read_write(lambda_joiner)
data_bucket.grant_read_write(lambda_joiner)
# trigger for 8th and 23rd of the month at 18:00 UTC (1pm EST)
lambda_joiner_rule = events.Rule(self,
"Lambda Joiner Rule",
schedule=events.Schedule.cron(
minute='0',
hour='18',
day="8,23",
month='*',
year='*'))
lambda_joiner_rule.add_target(targets.LambdaFunction(lambda_joiner))
def __init__(self, scope: core.Construct, stack_id: str, *,
env: core.Environment,
api_method: apigw.HttpMethod,
api_path: str,
function_code: lambda_.Code,
function_dependencies_layer: lambda_.LayerVersion,
function_name_prefix: str,
handler_function_handler: str,
receiver_function_handler: str,
queue_name: str,
**kwargs):
super().__init__(scope, stack_id, env=env, **kwargs)
# create the queue
self.queue = sqs.Queue(
self, 'Queue',
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=1,
queue=sqs.Queue(
self,
f'DLQ',
queue_name=f'{queue_name}-dlq')),
queue_name=queue_name)
# create the receiver function
# add the queue url as an environment variable
self.receiver_function = lambda_.Function(
self, 'ReceiverFunction',
code=function_code,
environment={'QUEUE_URL': self.queue.queue_url},
function_name=f'{function_name_prefix}-receiver',
handler=receiver_function_handler,
layers=[function_dependencies_layer],
# memory_size=256,
runtime=lambda_.Runtime.PYTHON_3_8)
# allow the receiver function to enqueue messages
self.queue.grant_send_messages(self.receiver_function)
# route requests to the receiver lambda
# (with a circular dependency, so never mind)
# api.add_routes(
# integration=apigw.LambdaProxyIntegration(
# handler=self.receiver_function),
# methods=[api_method],
# path=api_path)
# route requests to the receiver lambda
# (without creating a circular dependency?)
# integration = apigw.CfnIntegration(
# self, 'Integration',
# api_id=api.http_api_id,
# integration_type='AWS_PROXY',
# integration_uri=self.receiver_function.function_arn,
# payload_format_version='2.0')
# apigw.CfnRoute(self, 'Route',
# api_id=api.http_api_id,
# route_key=f'{api_method.value} {api_path}',
# target=f'integrations/{integration.ref}')
# # trigger the lambda with those routed requests
# lambda_.CfnEventSourceMapping(
# self, 'Mappping',
# event_source_arn=f'arn:aws:execute-api:{env.region}:{env.account}:{api.http_api_id}/*/*{api_path}',
# function_name=self.receiver_function.function_arn)
# create the handler function
self.handler_function = lambda_.Function(
self, 'HandlerFunction',
code=function_code,
function_name=f'{function_name_prefix}-handler',
handler=handler_function_handler,
layers=[function_dependencies_layer],
# memory_size=256,
runtime=lambda_.Runtime.PYTHON_3_8)
# add the queue as a trigger for the handler function
self.handler_function.add_event_source(SqsEventSource(self.queue))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
#IAM Roles
sqs_lambda_poller_role = iam.Role(
self,
id="SQSPollerRole",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaSQSQueueExecutionRole")
])
sqs_lambda_poller_role.add_to_policy(
iam.PolicyStatement(actions=["sqs:*", "states:*"],
effect=iam.Effect.ALLOW,
resources=["*"]))
sqs_delete_role = iam.Role(
self,
id="RemoveMessageFromQueueRole",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole")
])
sqs_delete_role.add_to_policy(
iam.PolicyStatement(actions=["sqs:*"],
effect=iam.Effect.ALLOW,
resources=["*"]))
#define the queue
dl_queue = sqs.Queue(
self,
"DeadLetterQueue",
)
dl_letter = sqs.DeadLetterQueue(max_receive_count=3, queue=dl_queue)
queue = sqs.Queue(self,
"TextractPipelineCdkQueue",
visibility_timeout=core.Duration.seconds(300),
dead_letter_queue=dl_letter)
queue.grant_consume_messages(sqs_lambda_poller_role)
#lamba policy
pdf2imageStatement = iam.PolicyStatement(actions=["s3:*"],
effect=iam.Effect.ALLOW,
resources=["*"])
callTextractStatement = iam.PolicyStatement(
actions=["textract:*", "s3:*"],
effect=iam.Effect.ALLOW,
resources=["*"])
callSQSstatement = iam.PolicyStatement(actions=["sqs:*", "states:*"],
effect=iam.Effect.ALLOW,
resources=["*"])
#set the lambda code paths
layer_files = os.path.join(os.path.dirname(__file__), os.pardir,
'vendor')
#define a layer for libraries
poppler_layer = lambda_.LayerVersion(
self,
"poppler_layer",
code=lambda_.Code.from_asset(path='layers/poppler_layer'),
compatible_runtimes=[lambda_.Runtime.PYTHON_3_7],
description="Poppler Lambda Layer")
#helper layers
textract_processor_layer = lambda_.LayerVersion(
self,
"HelperLayer",
code=lambda_.Code.from_asset(path='layers/helpers'),
compatible_runtimes=[lambda_.Runtime.PYTHON_3_7],
description="Helper Layers")
#define functions
#get message from sqs
poll_sqs_queue = lambda_.Function(
self,
"PollSQS",
runtime=lambda_.Runtime.PYTHON_3_7,
handler="process_sqs.handler",
code=lambda_.Code.from_asset("lambda_funcs/process_sqs"),
timeout=core.Duration.seconds(300),
tracing=lambda_.Tracing.ACTIVE,
role=sqs_lambda_poller_role,
environment={
"QueueUrl": queue.queue_url,
"ACCOUNT_ID": core.Aws.ACCOUNT_ID
},
)
#cron to trigger lambda
polling_rule = events.Rule(self,
"PollSQSRule",
schedule=events.Schedule.rate(
core.Duration.minutes(3)))
polling_rule.add_target(target=targets.LambdaFunction(poll_sqs_queue))
#process pdf
pdf2image = lambda_.Function(
self,
"Pdf2Image",
code=lambda_.Code.from_asset(
'lambda_funcs/pdf2image',
bundling={
"image":
lambda_.Runtime.PYTHON_3_7.bundling_docker_image,
"command": [
"bash", "-c",
"pip install -r requirements.txt -t /asset-input && cp -au . /asset-output"
]
}),
handler='process_images.handler',
runtime=lambda_.Runtime.PYTHON_3_7,
layers=[poppler_layer],
tracing=lambda_.Tracing.ACTIVE,
memory_size=1028,
timeout=core.Duration.seconds(300)).add_to_role_policy(
statement=pdf2imageStatement)
#call texract
callTextract = lambda_.Function(
self,
"CallTextract",
code=lambda_.Code.from_asset('lambda_funcs/calltextract'),
handler='calltextract.handler',
runtime=lambda_.Runtime.PYTHON_3_7,
layers=[textract_processor_layer],
tracing=lambda_.Tracing.ACTIVE,
memory_size=1028,
timeout=core.Duration.seconds(300)).add_to_role_policy(
statement=callTextractStatement)
#Remove message from queue
remove_message_from_queue = lambda_.Function(
self,
id="RemoveMessageFromQueue",
handler="remove_message.handler",
code=lambda_.Code.from_asset("lambda_funcs/remove_msg_func"),
runtime=lambda_.Runtime.PYTHON_3_7,
tracing=lambda_.Tracing.ACTIVE,
timeout=core.Duration.seconds(300),
role=sqs_delete_role,
environment={"QueueUrl": queue.queue_url})
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
if self.node.try_get_context('vpc_type'):
validate_cdk_json(self)
ES_LOADER_TIMEOUT = 600
######################################################################
# REGION mapping / ELB & Lambda Arch
######################################################################
elb_id_temp = region_info.FactName.ELBV2_ACCOUNT
elb_map_temp = region_info.RegionInfo.region_map(elb_id_temp)
region_dict = {}
for region in elb_map_temp:
# ELB account ID
region_dict[region] = {'ElbV2AccountId': elb_map_temp[region]}
# Lambda Arch
if region in ('us-east-1', 'us-east-2', 'us-west-2', 'ap-south-1',
'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1',
'eu-central-1', 'eu-west-1', 'eu-west-2'):
region_dict[region]['LambdaArch'] = (
aws_lambda.Architecture.ARM_64.name)
else:
region_dict[region]['LambdaArch'] = (
aws_lambda.Architecture.X86_64.name)
region_mapping = core.CfnMapping(
scope=self, id='RegionMap', mapping=region_dict)
######################################################################
# get params
######################################################################
allow_source_address = core.CfnParameter(
self, 'AllowedSourceIpAddresses', allowed_pattern=r'^[0-9./\s]*',
description='Space-delimited list of CIDR blocks',
default='10.0.0.0/8 172.16.0.0/12 192.168.0.0/16')
sns_email = core.CfnParameter(
self, 'SnsEmail', allowed_pattern=r'^[0-9a-zA-Z@_\-\+\.]*',
description=('Input your email as SNS topic, where Amazon '
'OpenSearch Service will send alerts to'),
default='*****@*****.**')
geoip_license_key = core.CfnParameter(
self, 'GeoLite2LicenseKey', allowed_pattern=r'^[0-9a-zA-Z]{16}$',
default='xxxxxxxxxxxxxxxx',
description=("If you wolud like to enrich geoip locaiton such as "
"IP address's country, get a license key form MaxMind"
" and input the key. If you not, keep "
"xxxxxxxxxxxxxxxx"))
reserved_concurrency = core.CfnParameter(
self, 'ReservedConcurrency', default=10, type='Number',
description=('Input reserved concurrency. Increase this value if '
'there are steady logs delay despite no errors'))
aes_domain_name = self.node.try_get_context('aes_domain_name')
bucket = f'{aes_domain_name}-{core.Aws.ACCOUNT_ID}'
s3bucket_name_geo = f'{bucket}-geo'
s3bucket_name_log = f'{bucket}-log'
s3bucket_name_snapshot = f'{bucket}-snapshot'
# organizations / multiaccount
org_id = self.node.try_get_context('organizations').get('org_id')
org_mgmt_id = self.node.try_get_context(
'organizations').get('management_id')
org_member_ids = self.node.try_get_context(
'organizations').get('member_ids')
no_org_ids = self.node.try_get_context(
'no_organizations').get('aws_accounts')
# Overwrite default S3 bucket name as customer name
temp_geo = self.node.try_get_context('s3_bucket_name').get('geo')
if temp_geo:
s3bucket_name_geo = temp_geo
else:
print('Using default bucket names')
temp_log = self.node.try_get_context('s3_bucket_name').get('log')
if temp_log:
s3bucket_name_log = temp_log
elif org_id or no_org_ids:
s3bucket_name_log = f'{aes_domain_name}-{self.account}-log'
else:
print('Using default bucket names')
temp_snap = self.node.try_get_context('s3_bucket_name').get('snapshot')
if temp_snap:
s3bucket_name_snapshot = temp_snap
else:
print('Using default bucket names')
kms_cmk_alias = self.node.try_get_context('kms_cmk_alias')
if not kms_cmk_alias:
kms_cmk_alias = 'aes-siem-key'
print('Using default key alais')
######################################################################
# deploy VPC when context is defined as using VPC
######################################################################
# vpc_type is 'new' or 'import' or None
vpc_type = self.node.try_get_context('vpc_type')
if vpc_type == 'new':
is_vpc = True
vpc_cidr = self.node.try_get_context('new_vpc_nw_cidr_block')
subnet_cidr_mask = int(
self.node.try_get_context('new_vpc_subnet_cidr_mask'))
is_vpc = True
# VPC
vpc_aes_siem = aws_ec2.Vpc(
self, 'VpcAesSiem', cidr=vpc_cidr,
max_azs=3, nat_gateways=0,
subnet_configuration=[
aws_ec2.SubnetConfiguration(
subnet_type=aws_ec2.SubnetType.ISOLATED,
name='aes-siem-subnet', cidr_mask=subnet_cidr_mask)])
subnet1 = vpc_aes_siem.isolated_subnets[0]
subnets = [{'subnet_type': aws_ec2.SubnetType.ISOLATED}]
vpc_subnets = aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.ISOLATED)
vpc_aes_siem_opt = vpc_aes_siem.node.default_child.cfn_options
vpc_aes_siem_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
for subnet in vpc_aes_siem.isolated_subnets:
subnet_opt = subnet.node.default_child.cfn_options
subnet_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
elif vpc_type == 'import':
vpc_id = self.node.try_get_context('imported_vpc_id')
vpc_aes_siem = aws_ec2.Vpc.from_lookup(
self, 'VpcAesSiem', vpc_id=vpc_id)
subnet_ids = get_subnet_ids(self)
subnets = []
for number, subnet_id in enumerate(subnet_ids, 1):
obj_id = 'Subenet' + str(number)
subnet = aws_ec2.Subnet.from_subnet_id(self, obj_id, subnet_id)
subnets.append(subnet)
subnet1 = subnets[0]
vpc_subnets = aws_ec2.SubnetSelection(subnets=subnets)
if vpc_type:
is_vpc = True
# Security Group
sg_vpc_noinbound_aes_siem = aws_ec2.SecurityGroup(
self, 'AesSiemVpcNoinboundSecurityGroup',
security_group_name='aes-siem-noinbound-vpc-sg',
vpc=vpc_aes_siem)
sg_vpc_aes_siem = aws_ec2.SecurityGroup(
self, 'AesSiemVpcSecurityGroup',
security_group_name='aes-siem-vpc-sg',
vpc=vpc_aes_siem)
sg_vpc_aes_siem.add_ingress_rule(
peer=aws_ec2.Peer.ipv4(vpc_aes_siem.vpc_cidr_block),
connection=aws_ec2.Port.tcp(443),)
sg_vpc_opt = sg_vpc_aes_siem.node.default_child.cfn_options
sg_vpc_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# VPC Endpoint
vpc_aes_siem.add_gateway_endpoint(
'S3Endpoint', service=aws_ec2.GatewayVpcEndpointAwsService.S3,
subnets=subnets)
vpc_aes_siem.add_interface_endpoint(
'SQSEndpoint', security_groups=[sg_vpc_aes_siem],
service=aws_ec2.InterfaceVpcEndpointAwsService.SQS,)
vpc_aes_siem.add_interface_endpoint(
'KMSEndpoint', security_groups=[sg_vpc_aes_siem],
service=aws_ec2.InterfaceVpcEndpointAwsService.KMS,)
else:
is_vpc = False
is_vpc = core.CfnCondition(
self, 'IsVpc', expression=core.Fn.condition_equals(is_vpc, True))
"""
CloudFormation実行時の条件式の書き方
ClassのBasesが aws_cdk.core.Resource の時は、
node.default_child.cfn_options.condition = is_vpc
ClassのBasesが aws_cdk.core.CfnResource の時は、
cfn_options.condition = is_vpc
"""
######################################################################
# create cmk of KMS to encrypt S3 bucket
######################################################################
kms_aes_siem = aws_kms.Key(
self, 'KmsAesSiemLog', description='CMK for SIEM solution',
removal_policy=core.RemovalPolicy.RETAIN)
aws_kms.Alias(
self, 'KmsAesSiemLogAlias', alias_name=kms_cmk_alias,
target_key=kms_aes_siem,
removal_policy=core.RemovalPolicy.RETAIN)
kms_aes_siem.add_to_resource_policy(
aws_iam.PolicyStatement(
sid='Allow GuardDuty to use the key',
actions=['kms:GenerateDataKey'],
principals=[aws_iam.ServicePrincipal(
'guardduty.amazonaws.com')],
resources=['*'],),)
kms_aes_siem.add_to_resource_policy(
aws_iam.PolicyStatement(
sid='Allow VPC Flow Logs to use the key',
actions=['kms:Encrypt', 'kms:Decrypt', 'kms:ReEncrypt*',
'kms:GenerateDataKey*', 'kms:DescribeKey'],
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
resources=['*'],),)
# basic policy
key_policy_basic1 = aws_iam.PolicyStatement(
sid='Allow principals in the account to decrypt log files',
actions=['kms:DescribeKey', 'kms:ReEncryptFrom'],
principals=[aws_iam.AccountPrincipal(
account_id=core.Aws.ACCOUNT_ID)],
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_basic1)
# for Athena
key_policy_athena = aws_iam.PolicyStatement(
sid='Allow Athena to query s3 objects with this key',
actions=['kms:Decrypt', 'kms:DescribeKey', 'kms:Encrypt',
'kms:GenerateDataKey*', 'kms:ReEncrypt*'],
principals=[aws_iam.AccountPrincipal(
account_id=core.Aws.ACCOUNT_ID)],
resources=['*'],
conditions={'ForAnyValue:StringEquals': {
'aws:CalledVia': 'athena.amazonaws.com'}})
kms_aes_siem.add_to_resource_policy(key_policy_athena)
# for CloudTrail
key_policy_trail1 = aws_iam.PolicyStatement(
sid='Allow CloudTrail to describe key',
actions=['kms:DescribeKey'],
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_trail1)
key_policy_trail2 = aws_iam.PolicyStatement(
sid=('Allow CloudTrail to encrypt logs'),
actions=['kms:GenerateDataKey*'],
principals=[aws_iam.ServicePrincipal(
'cloudtrail.amazonaws.com')],
resources=['*'],
conditions={'StringLike': {
'kms:EncryptionContext:aws:cloudtrail:arn': [
f'arn:aws:cloudtrail:*:{core.Aws.ACCOUNT_ID}:trail/*']}})
kms_aes_siem.add_to_resource_policy(key_policy_trail2)
######################################################################
# create s3 bucket
######################################################################
block_pub = aws_s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True
)
s3_geo = aws_s3.Bucket(
self, 'S3BucketForGeoip', block_public_access=block_pub,
bucket_name=s3bucket_name_geo,
# removal_policy=core.RemovalPolicy.DESTROY,
)
# create s3 bucket for log collector
s3_log = aws_s3.Bucket(
self, 'S3BucketForLog', block_public_access=block_pub,
bucket_name=s3bucket_name_log, versioned=True,
encryption=aws_s3.BucketEncryption.S3_MANAGED,
# removal_policy=core.RemovalPolicy.DESTROY,
)
# create s3 bucket for aes snapshot
s3_snapshot = aws_s3.Bucket(
self, 'S3BucketForSnapshot', block_public_access=block_pub,
bucket_name=s3bucket_name_snapshot,
# removal_policy=core.RemovalPolicy.DESTROY,
)
######################################################################
# IAM Role
######################################################################
# delopyment policy for lambda deploy-aes
arn_prefix = f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}'
loggroup_aes = f'log-group:/aws/aes/domains/{aes_domain_name}/*'
loggroup_opensearch = (
f'log-group:/aws/OpenSearchService/domains/{aes_domain_name}/*')
loggroup_lambda = 'log-group:/aws/lambda/aes-siem-*'
policydoc_create_loggroup = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=[
'logs:PutResourcePolicy',
'logs:DescribeLogGroups',
'logs:DescribeLogStreams'
],
resources=[f'{arn_prefix}:*', ]
),
aws_iam.PolicyStatement(
actions=[
'logs:CreateLogGroup', 'logs:CreateLogStream',
'logs:PutLogEvents', 'logs:PutRetentionPolicy'],
resources=[
f'{arn_prefix}:{loggroup_aes}',
f'{arn_prefix}:{loggroup_opensearch}',
f'{arn_prefix}:{loggroup_lambda}',
],
)
]
)
policydoc_crhelper = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=[
'lambda:AddPermission',
'lambda:RemovePermission',
'events:ListRules',
'events:PutRule',
'events:DeleteRule',
'events:PutTargets',
'events:RemoveTargets'],
resources=['*']
)
]
)
# snaphot rule for AES
policydoc_snapshot = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=['s3:ListBucket'],
resources=[s3_snapshot.bucket_arn]
),
aws_iam.PolicyStatement(
actions=['s3:GetObject', 's3:PutObject',
's3:DeleteObject'],
resources=[s3_snapshot.bucket_arn + '/*']
)
]
)
aes_siem_snapshot_role = aws_iam.Role(
self, 'AesSiemSnapshotRole',
role_name='aes-siem-snapshot-role',
inline_policies=[policydoc_snapshot, ],
assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com')
)
policydoc_assume_snapshrole = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=['iam:PassRole'],
resources=[aes_siem_snapshot_role.role_arn]
),
]
)
aes_siem_deploy_role_for_lambda = aws_iam.Role(
self, 'AesSiemDeployRoleForLambda',
role_name='aes-siem-deploy-role-for-lambda',
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonOpenSearchServiceFullAccess'),
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
],
inline_policies=[policydoc_assume_snapshrole, policydoc_snapshot,
policydoc_create_loggroup, policydoc_crhelper],
assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com')
)
if vpc_type:
aes_siem_deploy_role_for_lambda.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaVPCAccessExecutionRole')
)
# for alert from Amazon OpenSearch Service
aes_siem_sns_role = aws_iam.Role(
self, 'AesSiemSnsRole',
role_name='aes-siem-sns-role',
assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com')
)
# EC2 role
aes_siem_es_loader_ec2_role = aws_iam.Role(
self, 'AesSiemEsLoaderEC2Role',
role_name='aes-siem-es-loader-for-ec2',
assumed_by=aws_iam.ServicePrincipal('ec2.amazonaws.com'),
)
aws_iam.CfnInstanceProfile(
self, 'AesSiemEsLoaderEC2InstanceProfile',
instance_profile_name=aes_siem_es_loader_ec2_role.role_name,
roles=[aes_siem_es_loader_ec2_role.role_name]
)
######################################################################
# in VPC
######################################################################
aes_role_exist = check_iam_role('/aws-service-role/es.amazonaws.com/')
if vpc_type and not aes_role_exist:
slr_aes = aws_iam.CfnServiceLinkedRole(
self, 'AWSServiceRoleForAmazonOpenSearchService',
aws_service_name='es.amazonaws.com',
description='Created by cloudformation of siem stack'
)
slr_aes.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
######################################################################
# SQS for es-laoder's DLQ
######################################################################
sqs_aes_siem_dlq = aws_sqs.Queue(
self, 'AesSiemDlq', queue_name='aes-siem-dlq',
retention_period=core.Duration.days(14))
sqs_aes_siem_splitted_logs = aws_sqs.Queue(
self, 'AesSiemSqsSplitLogs',
queue_name='aes-siem-sqs-splitted-logs',
dead_letter_queue=aws_sqs.DeadLetterQueue(
max_receive_count=2, queue=sqs_aes_siem_dlq),
visibility_timeout=core.Duration.seconds(ES_LOADER_TIMEOUT),
retention_period=core.Duration.days(14))
######################################################################
# Setup Lambda
######################################################################
# setup lambda of es_loader
lambda_es_loader_vpc_kwargs = {}
if vpc_type:
lambda_es_loader_vpc_kwargs = {
'security_group': sg_vpc_noinbound_aes_siem,
'vpc': vpc_aes_siem,
'vpc_subnets': vpc_subnets,
}
lambda_es_loader = aws_lambda.Function(
self, 'LambdaEsLoader', **lambda_es_loader_vpc_kwargs,
function_name='aes-siem-es-loader',
description=f'{SOLUTION_NAME} / es-loader',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
# code=aws_lambda.Code.asset('../lambda/es_loader.zip'),
code=aws_lambda.Code.asset('../lambda/es_loader'),
handler='index.lambda_handler',
memory_size=2048,
timeout=core.Duration.seconds(ES_LOADER_TIMEOUT),
reserved_concurrent_executions=(
reserved_concurrency.value_as_number),
dead_letter_queue_enabled=True,
dead_letter_queue=sqs_aes_siem_dlq,
environment={
'GEOIP_BUCKET': s3bucket_name_geo, 'LOG_LEVEL': 'info',
'POWERTOOLS_LOGGER_LOG_EVENT': 'false',
'POWERTOOLS_SERVICE_NAME': 'es-loader',
'POWERTOOLS_METRICS_NAMESPACE': 'SIEM'})
es_loader_newver = lambda_es_loader.add_version(
name=__version__, description=__version__)
es_loader_opt = es_loader_newver.node.default_child.cfn_options
es_loader_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# send only
# sqs_aes_siem_dlq.grant(lambda_es_loader, 'sqs:SendMessage')
# send and reieve. but it must be loop
sqs_aes_siem_dlq.grant(
lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage',
'sqs:DeleteMessage', 'sqs:GetQueueAttributes')
sqs_aes_siem_splitted_logs.grant(
lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage',
'sqs:DeleteMessage', 'sqs:GetQueueAttributes')
lambda_es_loader.add_event_source(
aws_lambda_event_sources.SqsEventSource(
sqs_aes_siem_splitted_logs, batch_size=1))
# es-loaer on EC2 role
sqs_aes_siem_dlq.grant(
aes_siem_es_loader_ec2_role, 'sqs:GetQueue*', 'sqs:ListQueues*',
'sqs:ReceiveMessage*', 'sqs:DeleteMessage*')
lambda_geo = aws_lambda.Function(
self, 'LambdaGeoipDownloader',
function_name='aes-siem-geoip-downloader',
description=f'{SOLUTION_NAME} / geoip-downloader',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
code=aws_lambda.Code.asset('../lambda/geoip_downloader'),
handler='index.lambda_handler',
memory_size=320,
timeout=core.Duration.seconds(300),
environment={
's3bucket_name': s3bucket_name_geo,
'license_key': geoip_license_key.value_as_string,
}
)
lambda_geo_newver = lambda_geo.add_version(
name=__version__, description=__version__)
lamba_geo_opt = lambda_geo_newver.node.default_child.cfn_options
lamba_geo_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
######################################################################
# setup OpenSearch Service
######################################################################
lambda_deploy_es = aws_lambda.Function(
self, 'LambdaDeployAES',
function_name='aes-siem-deploy-aes',
description=f'{SOLUTION_NAME} / opensearch domain deployment',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
# code=aws_lambda.Code.asset('../lambda/deploy_es.zip'),
code=aws_lambda.Code.asset('../lambda/deploy_es'),
handler='index.aes_domain_handler',
memory_size=128,
timeout=core.Duration.seconds(300),
environment={
'accountid': core.Aws.ACCOUNT_ID,
'aes_domain_name': aes_domain_name,
'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn,
'es_loader_role': lambda_es_loader.role.role_arn,
'allow_source_address': allow_source_address.value_as_string,
},
role=aes_siem_deploy_role_for_lambda,
)
lambda_deploy_es.add_environment(
's3_snapshot', s3_snapshot.bucket_name)
if vpc_type:
lambda_deploy_es.add_environment(
'vpc_subnet_id', subnet1.subnet_id)
lambda_deploy_es.add_environment(
'security_group_id', sg_vpc_aes_siem.security_group_id)
else:
lambda_deploy_es.add_environment('vpc_subnet_id', 'None')
lambda_deploy_es.add_environment('security_group_id', 'None')
deploy_es_newver = lambda_deploy_es.add_version(
name=__version__, description=__version__)
deploy_es_opt = deploy_es_newver.node.default_child.cfn_options
deploy_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# execute lambda_deploy_es to deploy Amaozon ES Domain
aes_domain = aws_cloudformation.CfnCustomResource(
self, 'AesSiemDomainDeployedR2',
service_token=lambda_deploy_es.function_arn,)
aes_domain.add_override('Properties.ConfigVersion', __version__)
es_endpoint = aes_domain.get_att('es_endpoint').to_string()
lambda_es_loader.add_environment('ES_ENDPOINT', es_endpoint)
lambda_es_loader.add_environment(
'SQS_SPLITTED_LOGS_URL', sqs_aes_siem_splitted_logs.queue_url)
lambda_configure_es_vpc_kwargs = {}
if vpc_type:
lambda_configure_es_vpc_kwargs = {
'security_group': sg_vpc_noinbound_aes_siem,
'vpc': vpc_aes_siem,
'vpc_subnets': aws_ec2.SubnetSelection(subnets=[subnet1, ]), }
lambda_configure_es = aws_lambda.Function(
self, 'LambdaConfigureAES', **lambda_configure_es_vpc_kwargs,
function_name='aes-siem-configure-aes',
description=f'{SOLUTION_NAME} / opensearch configuration',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
code=aws_lambda.Code.asset('../lambda/deploy_es'),
handler='index.aes_config_handler',
memory_size=128,
timeout=core.Duration.seconds(300),
environment={
'accountid': core.Aws.ACCOUNT_ID,
'aes_domain_name': aes_domain_name,
'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn,
'es_loader_role': lambda_es_loader.role.role_arn,
'allow_source_address': allow_source_address.value_as_string,
'es_endpoint': es_endpoint,
},
role=aes_siem_deploy_role_for_lambda,
)
lambda_configure_es.add_environment(
's3_snapshot', s3_snapshot.bucket_name)
if vpc_type:
lambda_configure_es.add_environment(
'vpc_subnet_id', subnet1.subnet_id)
lambda_configure_es.add_environment(
'security_group_id', sg_vpc_aes_siem.security_group_id)
else:
lambda_configure_es.add_environment('vpc_subnet_id', 'None')
lambda_configure_es.add_environment('security_group_id', 'None')
configure_es_newver = lambda_configure_es.add_version(
name=__version__, description=__version__)
configure_es_opt = configure_es_newver.node.default_child.cfn_options
configure_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
aes_config = aws_cloudformation.CfnCustomResource(
self, 'AesSiemDomainConfiguredR2',
service_token=lambda_configure_es.function_arn,)
aes_config.add_override('Properties.ConfigVersion', __version__)
aes_config.add_depends_on(aes_domain)
aes_config.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
es_arn = (f'arn:aws:es:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}'
f':domain/{aes_domain_name}')
# grant permission to es_loader role
inline_policy_to_load_entries_into_es = aws_iam.Policy(
self, 'aes-siem-policy-to-load-entries-to-es',
policy_name='aes-siem-policy-to-load-entries-to-es',
statements=[
aws_iam.PolicyStatement(
actions=['es:*'],
resources=[es_arn + '/*', ]),
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_to_load_entries_into_es)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_to_load_entries_into_es)
# grant additional permission to es_loader role
additional_kms_cmks = self.node.try_get_context('additional_kms_cmks')
if additional_kms_cmks:
inline_policy_access_to_additional_cmks = aws_iam.Policy(
self, 'access_to_additional_cmks',
policy_name='access_to_additional_cmks',
statements=[
aws_iam.PolicyStatement(
actions=['kms:Decrypt'],
resources=sorted(set(additional_kms_cmks))
)
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_access_to_additional_cmks)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_access_to_additional_cmks)
additional_buckets = self.node.try_get_context('additional_s3_buckets')
if additional_buckets:
buckets_list = []
for bucket in additional_buckets:
buckets_list.append(f'arn:aws:s3:::{bucket}')
buckets_list.append(f'arn:aws:s3:::{bucket}/*')
inline_policy_access_to_additional_buckets = aws_iam.Policy(
self, 'access_to_additional_buckets',
policy_name='access_to_additional_buckets',
statements=[
aws_iam.PolicyStatement(
actions=['s3:GetObject*', 's3:GetBucket*', 's3:List*'],
resources=sorted(set(buckets_list))
)
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_access_to_additional_buckets)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_access_to_additional_buckets)
kms_aes_siem.grant_decrypt(lambda_es_loader)
kms_aes_siem.grant_decrypt(aes_siem_es_loader_ec2_role)
######################################################################
# s3 notification and grant permisssion
######################################################################
s3_geo.grant_read_write(lambda_geo)
s3_geo.grant_read(lambda_es_loader)
s3_geo.grant_read(aes_siem_es_loader_ec2_role)
s3_log.grant_read(lambda_es_loader)
s3_log.grant_read(aes_siem_es_loader_ec2_role)
# create s3 notification for es_loader
notification = aws_s3_notifications.LambdaDestination(lambda_es_loader)
# assign notification for the s3 PUT event type
# most log system use PUT, but also CLB use POST & Multipart Upload
s3_log.add_event_notification(
aws_s3.EventType.OBJECT_CREATED, notification,
aws_s3.NotificationKeyFilter(prefix='AWSLogs/'))
# For user logs, not AWS logs
s3_log.add_event_notification(
aws_s3.EventType.OBJECT_CREATED, notification,
aws_s3.NotificationKeyFilter(prefix='UserLogs/'))
# Download geoip to S3 once by executing lambda_geo
get_geodb = aws_cloudformation.CfnCustomResource(
self, 'ExecLambdaGeoipDownloader',
service_token=lambda_geo.function_arn,)
get_geodb.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
# Download geoip every day at 6PM UTC
rule = aws_events.Rule(
self, 'CwlRuleLambdaGeoipDownloaderDilly',
schedule=aws_events.Schedule.rate(core.Duration.hours(12)))
rule.add_target(aws_events_targets.LambdaFunction(lambda_geo))
######################################################################
# bucket policy
######################################################################
s3_awspath = s3_log.bucket_arn + '/AWSLogs/' + core.Aws.ACCOUNT_ID
bucket_policy_common1 = aws_iam.PolicyStatement(
sid='ELB Policy',
principals=[aws_iam.AccountPrincipal(
account_id=region_mapping.find_in_map(
core.Aws.REGION, 'ElbV2AccountId'))],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],)
# NLB / ALB / R53resolver / VPC Flow Logs
bucket_policy_elb1 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryAclCheck For ALB NLB R53Resolver Flowlogs',
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
actions=['s3:GetBucketAcl', 's3:ListBucket'],
resources=[s3_log.bucket_arn],)
bucket_policy_elb2 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryWrite For ALB NLB R53Resolver Flowlogs',
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_common1)
s3_log.add_to_resource_policy(bucket_policy_elb1)
s3_log.add_to_resource_policy(bucket_policy_elb2)
# CloudTrail
bucket_policy_trail1 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryAclCheck For Cloudtrail',
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:GetBucketAcl'], resources=[s3_log.bucket_arn],)
bucket_policy_trail2 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryWrite For CloudTrail',
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_trail1)
s3_log.add_to_resource_policy(bucket_policy_trail2)
# GuardDuty
bucket_policy_gd1 = aws_iam.PolicyStatement(
sid='Allow GuardDuty to use the getBucketLocation operation',
principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')],
actions=['s3:GetBucketLocation'], resources=[s3_log.bucket_arn],)
bucket_policy_gd2 = aws_iam.PolicyStatement(
sid='Allow GuardDuty to upload objects to the bucket',
principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_log.bucket_arn + '/*'],)
bucket_policy_gd5 = aws_iam.PolicyStatement(
sid='Deny non-HTTPS access', effect=aws_iam.Effect.DENY,
actions=['s3:*'], resources=[s3_log.bucket_arn + '/*'],
conditions={'Bool': {'aws:SecureTransport': 'false'}})
bucket_policy_gd5.add_any_principal()
s3_log.add_to_resource_policy(bucket_policy_gd1)
s3_log.add_to_resource_policy(bucket_policy_gd2)
s3_log.add_to_resource_policy(bucket_policy_gd5)
# Config
bucket_policy_config1 = aws_iam.PolicyStatement(
sid='AWSConfig BucketPermissionsCheck and BucketExistenceCheck',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:GetBucketAcl', 's3:ListBucket'],
resources=[s3_log.bucket_arn],)
bucket_policy_config2 = aws_iam.PolicyStatement(
sid='AWSConfigBucketDelivery',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/Config/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_config1)
s3_log.add_to_resource_policy(bucket_policy_config2)
# geoip
bucket_policy_geo1 = aws_iam.PolicyStatement(
sid='Allow geoip downloader and es-loader to read/write',
principals=[lambda_es_loader.role, lambda_geo.role],
actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'],
resources=[s3_geo.bucket_arn + '/*'],)
s3_geo.add_to_resource_policy(bucket_policy_geo1)
# ES Snapshot
bucket_policy_snapshot = aws_iam.PolicyStatement(
sid='Allow ES to store snapshot',
principals=[aes_siem_snapshot_role],
actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'],
resources=[s3_snapshot.bucket_arn + '/*'],)
s3_snapshot.add_to_resource_policy(bucket_policy_snapshot)
######################################################################
# for multiaccount / organizaitons
######################################################################
if org_id or no_org_ids:
##################################################################
# KMS key policy for multiaccount / organizaitons
##################################################################
# for CloudTrail
cond_tail2 = self.make_resource_list(
path='arn:aws:cloudtrail:*:', tail=':trail/*',
keys=self.list_without_none(org_mgmt_id, no_org_ids))
key_policy_mul_trail2 = aws_iam.PolicyStatement(
sid=('Allow CloudTrail to encrypt logs for multiaccounts'),
actions=['kms:GenerateDataKey*'],
principals=[aws_iam.ServicePrincipal(
'cloudtrail.amazonaws.com')],
resources=['*'],
conditions={'StringLike': {
'kms:EncryptionContext:aws:cloudtrail:arn': cond_tail2}})
kms_aes_siem.add_to_resource_policy(key_policy_mul_trail2)
# for replicaiton
key_policy_rep1 = aws_iam.PolicyStatement(
sid=('Enable cross account encrypt access for S3 Cross Region '
'Replication'),
actions=['kms:Encrypt'],
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_rep1)
##################################################################
# Buckdet Policy for multiaccount / organizaitons
##################################################################
s3_log_bucket_arn = 'arn:aws:s3:::' + s3bucket_name_log
# for CloudTrail
s3_mulpaths = self.make_resource_list(
path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/*',
keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids))
bucket_policy_org_trail = aws_iam.PolicyStatement(
sid='AWSCloudTrailWrite for Multiaccounts / Organizations',
principals=[
aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:PutObject'], resources=s3_mulpaths,
conditions={'StringEquals': {
's3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_org_trail)
# config
s3_conf_multpaths = self.make_resource_list(
path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/Config/*',
keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids))
bucket_policy_mul_config2 = aws_iam.PolicyStatement(
sid='AWSConfigBucketDelivery',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:PutObject'], resources=s3_conf_multpaths,
conditions={'StringEquals': {
's3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_mul_config2)
# for replication
bucket_policy_rep1 = aws_iam.PolicyStatement(
sid='PolicyForDestinationBucket / Permissions on objects',
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
actions=['s3:ReplicateDelete', 's3:ReplicateObject',
's3:ReplicateTags', 's3:GetObjectVersionTagging',
's3:ObjectOwnerOverrideToBucketOwner'],
resources=[f'{s3_log_bucket_arn}/*'])
bucket_policy_rep2 = aws_iam.PolicyStatement(
sid='PolicyForDestinationBucket / Permissions on bucket',
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
actions=['s3:List*', 's3:GetBucketVersioning',
's3:PutBucketVersioning'],
resources=[f'{s3_log_bucket_arn}'])
s3_log.add_to_resource_policy(bucket_policy_rep1)
s3_log.add_to_resource_policy(bucket_policy_rep2)
######################################################################
# SNS topic for Amazon OpenSearch Service Alert
######################################################################
sns_topic = aws_sns.Topic(
self, 'SnsTopic', topic_name='aes-siem-alert',
display_name='AES SIEM')
sns_topic.add_subscription(aws_sns_subscriptions.EmailSubscription(
email_address=sns_email.value_as_string))
sns_topic.grant_publish(aes_siem_sns_role)
######################################################################
# output of CFn
######################################################################
kibanaurl = f'https://{es_endpoint}/_dashboards/'
kibanaadmin = aes_domain.get_att('kibanaadmin').to_string()
kibanapass = aes_domain.get_att('kibanapass').to_string()
core.CfnOutput(self, 'RoleDeploy', export_name='role-deploy',
value=aes_siem_deploy_role_for_lambda.role_arn)
core.CfnOutput(self, 'DashboardsUrl', export_name='dashboards-url',
value=kibanaurl)
core.CfnOutput(self, 'DashboardsPassword',
export_name='dashboards-pass', value=kibanapass,
description=('Please change the password in OpenSearch '
'Dashboards ASAP'))
core.CfnOutput(self, 'DashboardsAdminID',
export_name='dashboards-admin', value=kibanaadmin)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
"""Initializes the stack
:param scope: parent of this stack, scope within which resources defined here are accessible
:type file_loc: Optional[aws_cdk.core.Construct]
:param id: the id of the stack
:type id: Optional[str]
:param **kwargs: additional optional arguments
``description``:
a description of the stack (`Optional[str]`).
``env``
AWS environment (account/region) where this stack will be deployed (`Optional[aws_cdk.core.Environment]`).
``stack_name``
name with which to deploy the stack (`Optional[str]`).
``synthesizer``
synthesis method to use while deploying this stack (`Optional[aws_cdk.core.IStackSynthesizer]`).
``tags``
stack tags that will be applied to all taggable resources as well as the stack (`Optional[Mapping[str, str]]`).
``termination_protection``
whether to enable termination protection for this stack (`Optional[bool]`).
"""
super().__init__(scope, id, **kwargs)
metric_handler_dict, webhook_creator_dict = self.handle_parameters()
# timeout used for lambda and sqs, in seconds
lambda_timeout = 300
if self.node.try_get_context('lambda_timeout'):
lambda_timeout = int(self.node.try_get_context('lambda_timeout'))
dead_letter_queue = sqs.Queue(self,
'DeadLetterQueue',
queue_name='DeadLetterQueue')
webhook_queue = sqs.Queue(
self,
'WebhookQueue',
queue_name='WebhookQueue',
visibility_timeout=core.Duration.seconds(lambda_timeout),
dead_letter_queue=sqs.DeadLetterQueue(max_receive_count=3,
queue=dead_letter_queue))
metric_handler_dict['queue_url'] = webhook_queue.queue_url
metric_handler_management_role = self.create_lambda_role_and_policy(
'MetricHandlerManagementRole', [
'cloudwatch:GetDashboard', 'cloudwatch:GetMetricData',
'cloudwatch:ListDashboards', 'cloudwatch:PutDashboard',
'cloudwatch:PutMetricData', 'logs:CreateLogGroup',
'logs:CreateLogStream', 'logs:PutLogEvents',
'secretsmanager:GetSecretValue'
])
metric_handler_timeout = lambda_timeout
metric_handler_function = _lambda.Function(
self,
'MetricsHandler',
function_name='MetricsHandler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda_dir'),
handler='cloudwatch_dashboard_handler.handler',
role=metric_handler_management_role,
environment=metric_handler_dict,
timeout=core.Duration.seconds(metric_handler_timeout))
self.create_event_with_permissions(metric_handler_function)
# Connect SQS to Lambda
sqs_event_source = lambda_event_source.SqsEventSource(webhook_queue)
metric_handler_function.add_event_source(sqs_event_source)
apigw_webhook_url = self.create_and_integrate_apigw(
webhook_queue, metric_handler_dict['dashboard_name_prefix'])
webhook_creator_dict['apigw_endpoint'] = apigw_webhook_url
webhook_role = self.create_lambda_role_and_policy(
'WebhookCreatorRole', ['secretsmanager:GetSecretValue'])
_lambda.Function(self,
'WebhookCreator',
function_name='WebhookCreator',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda_dir'),
handler='webhook_creator.handler',
role=webhook_role,
environment=webhook_creator_dict,
timeout=core.Duration.seconds(5))
def __init__(
self,
scope: core.Construct,
id: str,
entrypoint: Optional[List] = None,
cpu: Union[int, float] = 256,
memory: Union[int, float] = 512,
mincount: int = 0,
maxcount: int = 50,
scaling_steps: int = 5,
permissions: Optional[List[iam.PolicyStatement]] = None,
vpc_id: Optional[str] = None,
vpc_is_default: Optional[bool] = None,
environment: dict = {},
**kwargs: Any,
) -> None:
"""Define stack."""
super().__init__(scope, id, **kwargs)
permissions = permissions or []
vpc = ec2.Vpc.from_lookup(self,
"vpc",
vpc_id=vpc_id,
is_default=vpc_is_default)
cluster = ecs.Cluster(self, f"{id}-cluster", vpc=vpc)
topic = sns.Topic(self,
"ecsTopic",
display_name="ECS Watchbot SNS Topic")
core.CfnOutput(
self,
"SNSTopic",
value=topic.topic_arn,
description="SNS Topic ARN",
export_name=f"{id}-SNSTopic",
)
dlqueue = sqs.Queue(self, "ecsDeadLetterQueue")
core.CfnOutput(
self,
"DeadSQSQueueURL",
value=dlqueue.queue_url,
description="DeadLetter SQS URL",
export_name=f"{id}-DeadSQSQueueURL",
)
queue = sqs.Queue(
self,
"ecsQueue",
dead_letter_queue=sqs.DeadLetterQueue(queue=dlqueue,
max_receive_count=3),
)
core.CfnOutput(
self,
"SQSQueueURL",
value=queue.queue_url,
description="SQS URL",
export_name=f"{id}-SQSQueueURL",
)
environment.update({
"REGION": self.region,
"QUEUE_NAME": queue.queue_name
})
topic.add_subscription(sns_sub.SqsSubscription(queue))
fargate_task_definition = ecs.FargateTaskDefinition(
self,
"FargateTaskDefinition",
memory_limit_mib=memory,
cpu=cpu,
)
log_driver = ecs.AwsLogDriver(
stream_prefix=f"/ecs/tilebot/{id}",
log_retention=aws_logs.RetentionDays.ONE_WEEK,
)
fargate_task_definition.add_container(
"FargateContainer",
image=ecs.ContainerImage.from_asset(directory="./"),
entry_point=entrypoint,
environment=environment,
logging=log_driver,
)
fargate_service = ecs.FargateService(
self,
"FargateService",
cluster=cluster,
task_definition=fargate_task_definition,
desired_count=mincount,
enable_ecs_managed_tags=True,
assign_public_ip=True,
)
permissions.append(
iam.PolicyStatement(
actions=["sqs:*"],
resources=[queue.queue_arn],
))
for perm in permissions:
fargate_service.task_definition.task_role.add_to_policy(perm)
total_number_of_message_lambda = aws_lambda.Function(
self,
f"{id}-TotalMessagesLambda",
description="Create TotalNumberOfMessage metrics",
code=aws_lambda.Code.from_inline("""const AWS = require('aws-sdk');
exports.handler = function(event, context, callback) {
const sqs = new AWS.SQS({ region: process.env.AWS_DEFAULT_REGION });
const cw = new AWS.CloudWatch({ region: process.env.AWS_DEFAULT_REGION });
return sqs.getQueueAttributes({
QueueUrl: process.env.SQS_QUEUE_URL,
AttributeNames: ['ApproximateNumberOfMessagesNotVisible', 'ApproximateNumberOfMessages']
}).promise()
.then((attrs) => {
return cw.putMetricData({
Namespace: 'AWS/SQS',
MetricData: [{
MetricName: 'TotalNumberOfMessages',
Dimensions: [{ Name: 'QueueName', Value: process.env.SQS_QUEUE_NAME }],
Value: Number(attrs.Attributes.ApproximateNumberOfMessagesNotVisible) +
Number(attrs.Attributes.ApproximateNumberOfMessages)
}]
}).promise();
})
.then((metric) => callback(null, metric))
.catch((err) => callback(err));
};"""),
handler="index.handler",
runtime=aws_lambda.Runtime.NODEJS_10_X,
timeout=core.Duration.seconds(60),
environment={
"SQS_QUEUE_URL": queue.queue_url,
"SQS_QUEUE_NAME": queue.queue_name,
},
)
total_number_of_message_lambda.add_to_role_policy(
iam.PolicyStatement(
actions=["sqs:GetQueueAttributes"],
resources=[queue.queue_arn],
))
total_number_of_message_lambda.add_to_role_policy(
iam.PolicyStatement(
actions=["cloudwatch:PutMetricData"],
resources=["*"],
))
total_number_of_message_lambda.add_to_role_policy(
iam.PolicyStatement(
actions=["logs:*"],
resources=["arn:aws:logs:*:*:*"],
))
rule = aws_events.Rule(
self,
"TotalMessagesSchedule",
schedule=aws_events.Schedule.rate(core.Duration.seconds(60)),
)
rule.add_target(
aws_events_targets.LambdaFunction(total_number_of_message_lambda))
scalable_target = auto_scale.ScalableTarget(
self,
"AutoScallingTarget",
min_capacity=mincount,
max_capacity=maxcount,
service_namespace=auto_scale.ServiceNamespace.ECS,
resource_id="/".join([
"service", cluster.cluster_name, fargate_service.service_name
]),
scalable_dimension="ecs:service:DesiredCount",
)
scalable_target.node.add_dependency(fargate_service)
scale_up = auto_scale.CfnScalingPolicy(
self,
"ScaleUp",
policy_name="PolicyScaleUp",
policy_type="StepScaling",
scaling_target_id=scalable_target.scalable_target_id,
step_scaling_policy_configuration=auto_scale.CfnScalingPolicy.
StepScalingPolicyConfigurationProperty(
adjustment_type="ChangeInCapacity",
cooldown=120,
metric_aggregation_type="Maximum",
step_adjustments=[
auto_scale.CfnScalingPolicy.StepAdjustmentProperty(
scaling_adjustment=scaling_steps,
metric_interval_lower_bound=0,
),
],
),
)
scale_up_trigger = aws_cloudwatch.CfnAlarm( # noqa
self,
"ScaleUpTrigger",
alarm_description="Scale up due to visible messages in queue",
dimensions=[
aws_cloudwatch.CfnAlarm.DimensionProperty(
name="QueueName",
value=queue.queue_name,
),
],
metric_name="ApproximateNumberOfMessagesVisible",
namespace="AWS/SQS",
evaluation_periods=1,
comparison_operator="GreaterThanThreshold",
period=60,
statistic="Maximum",
threshold=0,
alarm_actions=[scale_up.ref],
)
scale_down = auto_scale.CfnScalingPolicy(
self,
"ScaleDown",
policy_name="PolicyScaleDown",
policy_type="StepScaling",
scaling_target_id=scalable_target.scalable_target_id,
step_scaling_policy_configuration=auto_scale.CfnScalingPolicy.
StepScalingPolicyConfigurationProperty(
adjustment_type="ExactCapacity",
cooldown=60,
step_adjustments=[
auto_scale.CfnScalingPolicy.StepAdjustmentProperty(
scaling_adjustment=mincount,
metric_interval_upper_bound=0,
),
],
),
)
scale_down_trigger = aws_cloudwatch.CfnAlarm( # noqa
self,
"ScaleDownTrigger",
alarm_description=
"Scale down due to lack of in-flight messages in queue",
dimensions=[
aws_cloudwatch.CfnAlarm.DimensionProperty(
name="QueueName",
value=queue.queue_name,
),
],
metric_name="TotalNumberOfMessages",
namespace="AWS/SQS",
evaluation_periods=1,
comparison_operator="LessThanThreshold",
period=120,
statistic="Maximum",
threshold=1,
alarm_actions=[scale_down.ref],
)
def __init__(self, scope: core.Construct, construct_id: str,
stack_log_level: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Add your stack resources below):
# Maximum number of times, a message can be tried to be process from the queue before deleting
self.max_msg_receive_cnt = 5
self.max_msg_receive_cnt_at_retry = 3
# Define Dead Letter Queue
self.reliable_q_dlq = _sqs.Queue(
self,
"DeadLetterQueue",
delivery_delay=core.Duration.seconds(100),
queue_name=f"reliable_q_dlq",
retention_period=core.Duration.days(2),
visibility_timeout=core.Duration.seconds(10),
receive_message_wait_time=core.Duration.seconds(10))
# Define Retry Queue for Reliable Q
self.reliable_q_retry_1 = _sqs.Queue(
self,
"reliableQueueRetry1",
delivery_delay=core.Duration.seconds(10),
queue_name=f"reliable_q_retry_1",
retention_period=core.Duration.days(2),
visibility_timeout=core.Duration.seconds(10),
receive_message_wait_time=core.Duration.seconds(10),
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=self.max_msg_receive_cnt_at_retry,
queue=self.reliable_q_dlq))
# Primary Source Queue
self.reliable_q = _sqs.Queue(
self,
"reliableQueue",
delivery_delay=core.Duration.seconds(5),
queue_name=f"reliable_q",
retention_period=core.Duration.days(2),
visibility_timeout=core.Duration.seconds(10),
receive_message_wait_time=core.Duration.seconds(10),
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=self.max_msg_receive_cnt,
queue=self.reliable_q_retry_1))
########################################
####### #######
####### SQS Data Producer #######
####### #######
########################################
# Read Lambda Code
try:
with open(
"stacks/back_end/serverless_sqs_producer_stack/lambda_src/sqs_data_producer.py",
encoding="utf-8",
mode="r") as f:
data_producer_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
data_producer_fn = _lambda.Function(
self,
"sqsDataProducerFn",
function_name=f"data_producer_fn_{construct_id}",
description="Produce data events and push to SQS",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(data_producer_fn_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(5),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": f"{stack_log_level}",
"APP_ENV": "Production",
"RELIABLE_QUEUE_NAME": f"{self.reliable_q.queue_name}",
"TRIGGER_RANDOM_FAILURES": "True"
})
# Grant our Lambda Producer privileges to write to SQS
self.reliable_q.grant_send_messages(data_producer_fn)
# Create Custom Loggroup for Producer
data_producer_lg = _logs.LogGroup(
self,
"dataProducerLogGroup",
log_group_name=f"/aws/lambda/{data_producer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
# Restrict Produce Lambda to be invoked only from the stack owner account
data_producer_fn.add_permission(
"restrictLambdaInvocationToFhInOwnAccount",
principal=_iam.AccountRootPrincipal(),
action="lambda:InvokeFunction",
source_account=core.Aws.ACCOUNT_ID)
# Monitoring for Queue
reliable_q_alarm = _cw.Alarm(
self,
"reliableQueueAlarm",
metric=self.reliable_q.metric(
"ApproximateNumberOfMessagesVisible"),
statistic="sum",
threshold=10,
period=core.Duration.minutes(5),
evaluation_periods=1,
comparison_operator=_cw.ComparisonOperator.GREATER_THAN_THRESHOLD)
###########################################
################# OUTPUTS #################
###########################################
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
"SqsDataProducer",
value=
f"https://console.aws.amazon.com/lambda/home?region={core.Aws.REGION}#/functions/{data_producer_fn.function_name}",
description="Produce data events and push to SQS Queue.")
output_2 = core.CfnOutput(
self,
"ReliableQueue",
value=
f"https://console.aws.amazon.com/sqs/v2/home?region={core.Aws.REGION}#/queues",
description="Reliable Queue")
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
ddb_file_list = ddb.Table(self, "ddb",
partition_key=ddb.Attribute(name="Key", type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
sqs_queue_DLQ = sqs.Queue(self, "sqs_DLQ",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14)
)
sqs_queue = sqs.Queue(self, "sqs_queue",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=100,
queue=sqs_queue_DLQ
)
)
handler = lam.Function(self, "lambdaFunction",
code=lam.Code.asset("./lambda"),
handler="lambda_function.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name': ddb_file_list.table_name,
'Des_bucket_default': Des_bucket_default,
'Des_prefix_default': Des_prefix_default,
'StorageClass': StorageClass,
'aws_access_key_id': aws_access_key_id,
'aws_secret_access_key': aws_secret_access_key,
'aws_access_key_region': aws_access_key_region
})
ddb_file_list.grant_read_write_data(handler)
handler.add_event_source(SqsEventSource(sqs_queue))
s3bucket = s3.Bucket(self, "s3bucket")
s3bucket.grant_read(handler)
s3bucket.add_event_notification(s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(sqs_queue))
# You can import an existing bucket and grant access to lambda
# exist_s3bucket = s3.Bucket.from_bucket_name(self, "import_bucket",
# bucket_name="you_bucket_name")
# exist_s3bucket.grant_read(handler)
# But You have to add sqs as imported bucket event notification manually, it doesn't support by CloudFormation
# An work around is to add on_cloud_trail_event for the bucket, but will trigger could_trail first
# 因为是导入的Bucket，需要手工建Bucket Event Trigger SQS，以及设置SQS允许该bucekt触发的Permission
core.CfnOutput(self, "DynamoDB_Table", value=ddb_file_list.table_name)
core.CfnOutput(self, "SQS_Job_Queue", value=sqs_queue.queue_name)
core.CfnOutput(self, "SQS_Job_Queue_DLQ", value=sqs_queue_DLQ.queue_name)
core.CfnOutput(self, "Worker_Lambda_Function", value=handler.function_name)
core.CfnOutput(self, "New_S3_Bucket", value=s3bucket.bucket_name)
# Create Lambda logs filter to create network traffic metric
handler.log_group.add_metric_filter("Complete-bytes",
metric_name="Complete-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Complete", bytes, key]'))
handler.log_group.add_metric_filter("Uploading-bytes",
metric_name="Uploading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Uploading", bytes, key]'))
handler.log_group.add_metric_filter("Downloading-bytes",
metric_name="Downloading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Downloading", bytes, key]'))
lambda_metric_Complete = cw.Metric(namespace="s3_migrate",
metric_name="Complete-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Upload = cw.Metric(namespace="s3_migrate",
metric_name="Uploading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Download = cw.Metric(namespace="s3_migrate",
metric_name="Downloading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
handler.log_group.add_metric_filter("ERROR",
metric_name="ERROR-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal(
'"ERROR"'))
handler.log_group.add_metric_filter("WARNING",
metric_name="WARNING-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal(
'"WARNING"'))
log_metric_ERROR = cw.Metric(namespace="s3_migrate",
metric_name="ERROR-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
log_metric_WARNING = cw.Metric(namespace="s3_migrate",
metric_name="WARNING-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
# Dashboard to monitor SQS and Lambda
board = cw.Dashboard(self, "s3_migrate", dashboard_name="s3_migrate_serverless")
board.add_widgets(cw.GraphWidget(title="Lambda-NETWORK",
left=[lambda_metric_Download, lambda_metric_Upload, lambda_metric_Complete]),
# TODO: here monitor all lambda concurrency not just the working one. Limitation from CDK
# Lambda now supports monitor single lambda concurrency, will change this after CDK support
cw.GraphWidget(title="Lambda-all-concurrent",
left=[handler.metric_all_concurrent_executions(period=core.Duration.minutes(1))]),
cw.GraphWidget(title="Lambda-invocations/errors/throttles",
left=[handler.metric_invocations(period=core.Duration.minutes(1)),
handler.metric_errors(period=core.Duration.minutes(1)),
handler.metric_throttles(period=core.Duration.minutes(1))]),
cw.GraphWidget(title="Lambda-duration",
left=[handler.metric_duration(period=core.Duration.minutes(1))]),
)
board.add_widgets(cw.GraphWidget(title="SQS-Jobs",
left=[sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)
),
sqs_queue.metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)
)]),
cw.GraphWidget(title="SQS-DeadLetterQueue",
left=[sqs_queue_DLQ.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)
),
sqs_queue_DLQ.metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)
)]),
cw.GraphWidget(title="ERROR/WARNING Logs",
left=[log_metric_ERROR],
right=[log_metric_WARNING]),
cw.SingleValueWidget(title="Running/Waiting and Dead Jobs",
metrics=[sqs_queue.metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)
),
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)
),
sqs_queue_DLQ.metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)
),
sqs_queue_DLQ.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)
)],
height=6)
)
# Alarm for queue - DLQ
alarm_DLQ = cw.Alarm(self, "SQS_DLQ",
alarm_name="s3-migration-serverless-SQS Dead Letter Queue",
metric=sqs_queue_DLQ.metric_approximate_number_of_messages_visible(),
threshold=0,
comparison_operator=cw.ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluation_periods=1,
datapoints_to_alarm=1)
alarm_topic = sns.Topic(self, "SQS queue-DLQ has dead letter")
alarm_topic.add_subscription(subscription=sub.EmailSubscription(alarm_email))
alarm_DLQ.add_alarm_action(action.SnsAction(alarm_topic))
# Alarm for queue empty, i.e. no visible message and no in-visible message
# metric_all_message = cw.MathExpression(
# expression="a + b",
# label="empty_queue_expression",
# using_metrics={
# "a": sqs_queue.metric_approximate_number_of_messages_visible(),
# "b": sqs_queue.metric_approximate_number_of_messages_not_visible()
# }
# )
# alarm_0 = cw.Alarm(self, "SQSempty",
# alarm_name="SQS queue empty-Serverless",
# metric=metric_all_message,
# threshold=0,
# comparison_operator=cw.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
# evaluation_periods=3,
# datapoints_to_alarm=3,
# treat_missing_data=cw.TreatMissingData.IGNORE
# )
# alarm_topic = sns.Topic(self, "SQS queue empty-Serverless")
# alarm_topic.add_subscription(subscription=sub.EmailSubscription(alarm_email))
# alarm_0.add_alarm_action(action.SnsAction(alarm_topic))
# core.CfnOutput(self, "Alarm", value="CloudWatch SQS queue empty Alarm for Serverless: " + alarm_email)
core.CfnOutput(self, "Dashboard", value="CloudWatch Dashboard name s3_migrate_serverless")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Tag all constructs with the project for easy billing drilldown,
# filtering, and organization.
core.Tags.of(self).add('project', 'MediaTranscription')
# Media files bucket
media_bucket = s3.Bucket(
self,
'media-transcription-bucket',
encryption=s3.BucketEncryption.S3_MANAGED,
)
# SQS queue for media files bucket event notifications
media_bucket_event_queue = sqs.Queue(
self,
'media-transcription-event-notification-queue',
queue_name='media-transcription-event-notification-queue',
visibility_timeout=core.Duration.seconds(60),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3,
queue=sqs.Queue(
self,
'media-transcription-event-notifications-dlq',
queue_name='media-transcription-event-notifications-dlq',
)),
)
# S3 object created notifications sent to SQS queue
media_bucket.add_event_notification(
s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(media_bucket_event_queue),
*[s3.NotificationKeyFilter(prefix='media-input/')],
)
# Lambda function to create/submit Transcribe jobs
transcribe_job_init_fn = lambda_.Function(
self,
'transcribe-job-init-fn',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.from_asset(
'../lambdas/transcribe-job-init-fn',
# The following is just dumb.
# The Lambda runtime doesn't use the latest boto3 by default.
# In order to use the latest boto3, we have to pip install
# and bundle locally using Docker.
# Q: Why need the latest boto3?
# A: https://github.com/boto/boto3/issues/2630
# I'll have to delete the ECR containers to avoid cost.
# TODO: Revert back to normal in like a month I guess.
bundling={
'image':
lambda_.Runtime.PYTHON_3_8.bundling_docker_image,
'command': [
'bash', '-c',
'\n pip install -r requirements.txt -t /asset-output &&\n cp -au . /asset-output\n '
]
}),
handler='fn.handler',
reserved_concurrent_executions=1, # Effectively single-threaded
)
# Triggered by SQS messages created for media file puts
transcribe_job_init_fn.add_event_source(
les.SqsEventSource(
queue=media_bucket_event_queue,
batch_size=5,
enabled=True,
))
# Grant access to start transcription jobs
transcribe_job_init_fn.add_to_role_policy(
statement=iam.PolicyStatement(
actions=[
'transcribe:StartTranscriptionJob',
],
resources=['*'],
effect=iam.Effect.ALLOW,
))
# Grant Lambda role to read and write to input and output portions of
# the S3 bucket.
# Q: Why grant Lambda the permissions instead of Transcribe service?
# A: Two-fold:
# - i) https://amzn.to/321Nx5I
# - ii) Granting just to this Lambda means other Transcribe jobs
# across the account cannot use this bucket (least privilege).
media_bucket.grant_read(
identity=transcribe_job_init_fn.grant_principal,
objects_key_pattern='media-input/*')
# Cannot specify a prefix for writes as Transcribe will not accept
# a job unless it has write permission on the whole bucket.
# Edit: The above statement was when I had to use '*' for writes. But
# now, I granted access to that .write_access_check_file.temp
# file and it seems to all work now?
media_bucket.grant_write(
identity=transcribe_job_init_fn.grant_principal,
objects_key_pattern='transcribe-output-raw/*')
# This is just as frustrating to you as it is to me.
media_bucket.grant_write(
identity=transcribe_job_init_fn.grant_principal,
objects_key_pattern='.write_access_check_file.temp')
# DynamoDB table for Jobs metadata
jobs_metadata_table = ddb.Table(
self,
'MediaTranscription-TranscriptionJobs',
table_name='MediaTranscription-TranscriptionJobs',
partition_key=ddb.Attribute(
name='Bucket-Key-ETag',
type=ddb.AttributeType.STRING,
),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST,
)
jobs_metadata_table.grant(transcribe_job_init_fn.grant_principal, *[
'dynamodb:GetItem',
'dynamodb:PutItem',
])
# Create IAM Group with read/write permissions to S3 bucket
# TODO: Make this more federated and robust
console_users_group = iam.Group(self, 'MediaTranscriptionConsoleUsers')
console_users_group.attach_inline_policy(policy=iam.Policy(
self,
'MediaTranscriptionConsoleUserS3Access',
statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
's3:ListBucket',
],
resources=[
media_bucket.bucket_arn,
],
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
's3:GetObject',
's3:PutObject',
],
resources=[
media_bucket.arn_for_objects('media-input/*'),
],
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
's3:GetObject',
],
resources=[
media_bucket.arn_for_objects(
'transcribe-output-raw/*'),
],
),
],
))
def __init__(self, scope: core.Construct, id: str, instance_id: str,
contact_flow_id: str, source_phone_number: str, timeout: int,
**kwargs):
super().__init__(scope, id, **kwargs)
web_bucket = _s3.Bucket(self,
"StaticWebBucket",
website_index_document="index.html",
website_error_document="index.html",
removal_policy=core.RemovalPolicy.DESTROY,
public_read_access=True)
core.CfnOutput(self,
'WebBucketUrl',
value=web_bucket.bucket_domain_name)
web_distribution = _clf.CloudFrontWebDistribution(
self,
'StaticWebDistribution',
origin_configs=[
_clf.SourceConfiguration(
s3_origin_source=_clf.S3OriginConfig(
s3_bucket_source=web_bucket),
behaviors=[_clf.Behavior(is_default_behavior=True)])
],
viewer_protocol_policy=_clf.ViewerProtocolPolicy.REDIRECT_TO_HTTPS)
_s3d.BucketDeployment(
self,
"S3StaticWebContentDeploymentWithInvalidation",
sources=[
_s3d.Source.asset(
f"{pathlib.Path(__file__).parent.absolute()}/site-content/build"
)
],
destination_bucket=web_bucket,
distribution=web_distribution,
distribution_paths=["/*"])
file_bucket = _s3.Bucket(self,
"FileBucket",
removal_policy=core.RemovalPolicy.DESTROY)
call_dead_letter_queue = _sqs.Queue(self,
"CallDeadLetterQueue",
fifo=True,
content_based_deduplication=True)
call_sqs_queue = _sqs.Queue(
self,
"CallSqsQueue",
fifo=True,
content_based_deduplication=True,
visibility_timeout=core.Duration.seconds(120),
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=1, queue=call_dead_letter_queue))
async_call_dead_letter_queue = _sqs.Queue(
self,
"AsyncCallDeadLetterQueue",
fifo=True,
content_based_deduplication=True)
async_callout_queue = _sqs.Queue(
self,
"AsyncCalloutQueue",
fifo=True,
content_based_deduplication=True,
visibility_timeout=core.Duration.seconds(120),
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=1, queue=async_call_dead_letter_queue))
call_job_complete_sns_topic = _sns.Topic(
self, "CallJobCompleteSnsTopic", display_name="CallJobCompletion")
call_result_table = _dynamodb.Table(
self,
"CallResultDynamodbTable",
billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST,
partition_key=_dynamodb.Attribute(
name="task_id", type=_dynamodb.AttributeType.STRING),
sort_key=_dynamodb.Attribute(name="receiver_id",
type=_dynamodb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.DESTROY)
callout_record_table = _dynamodb.Table(
self,
"CallTaskDynamodbTable",
billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST,
partition_key=_dynamodb.Attribute(
name="task_id", type=_dynamodb.AttributeType.STRING),
sort_key=_dynamodb.Attribute(name="created_at",
type=_dynamodb.AttributeType.NUMBER),
removal_policy=core.RemovalPolicy.DESTROY)
callout_record_table.add_global_secondary_index(
partition_key=_dynamodb.Attribute(
name='call_type', type=_dynamodb.AttributeType.STRING),
sort_key=_dynamodb.Attribute(name='created_at',
type=_dynamodb.AttributeType.NUMBER),
index_name='CallTypeCreatedAtGlobalIndex',
projection_type=_dynamodb.ProjectionType.ALL)
python_function_layer = _lambda.LayerVersion(
self,
"LambdaPythonFunctionLayer",
code=_lambda.Code.asset("aws_callouts_cdk/layer/_python"),
compatible_runtimes=[
_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8
],
license="Available under the MIT-0 license")
nodejs_function_layer = _lambda.LayerVersion(
self,
"LambdaNodeJsFunctionLayer",
code=_lambda.Code.asset("aws_callouts_cdk/layer/_nodejs"),
compatible_runtimes=[
_lambda.Runtime.NODEJS_10_X, _lambda.Runtime.NODEJS_12_X
],
license="Available under the MIT-0 license")
global_python_function_arguments = {
"code": _lambda.Code.asset("aws_callouts_cdk/src/python"),
"layers": [python_function_layer],
"runtime": _lambda.Runtime.PYTHON_3_7
}
global_nodeje_function_arguments = {
"code": _lambda.Code.asset("aws_callouts_cdk/src/nodejs"),
"layers": [nodejs_function_layer],
"runtime": _lambda.Runtime.NODEJS_12_X
}
get_callout_job_function = _lambda.Function(
self,
"GetCalloutJobFunction",
handler="get_call_job.lambda_handler",
**global_python_function_arguments)
get_callout_job_function.add_environment(key="S3Bucket",
value=file_bucket.bucket_name)
file_bucket.grant_read(get_callout_job_function)
callout_function = _lambda.Function(self,
"CalloutFunction",
handler="send_call.lambda_handler",
**global_python_function_arguments)
callout_function.add_environment(
key="ContactFlowArn",
value=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/contact-flow/{contact_flow_id}"
)
callout_function.add_environment(key="SourcePhoneNumber",
value=source_phone_number)
callout_function.add_environment(key="ExcelFileBucket",
value=file_bucket.bucket_name)
callout_function.add_environment(key="AsynCalloutQueueUrl",
value=async_callout_queue.queue_url)
callout_function.add_to_role_policy(statement=_iam.PolicyStatement(
resources=[
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/*"
],
actions=["connect:StartOutboundVoiceContact"]))
callout_function.add_event_source(source=_les.SqsEventSource(
queue=async_callout_queue, batch_size=1))
'arn:aws:connect:751225572132:ap-southeast-2:instance/9d0c7cc5-7d2a-42e4-a3dd-70f402e0d040'
file_bucket.grant_read_write(callout_function)
response_handler_function = _lambda.Function(
self,
"ResponseHandlerFunction",
handler="response_handler.lambda_handler",
**global_python_function_arguments)
response_handler_function.add_permission(
id="ResponseHandlerFunctionLambdaInvokePermission",
principal=_iam.ServicePrincipal(service="connect.amazonaws.com"),
action="lambda:InvokeFunction",
source_account=self.account,
source_arn=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}"
)
send_task_success_function = _lambda.Function(
self,
"SendTaskSuccessFunction",
handler="send_task_success.lambda_handler",
**global_python_function_arguments)
send_task_success_function.add_permission(
id="SendTaskSuccessFunctionLambdaInvokePermission",
principal=_iam.ServicePrincipal(service="connect.amazonaws.com"),
action="lambda:InvokeFunction",
source_account=self.account,
source_arn=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}"
)
get_call_result_function = _lambda.Function(
self,
"GetCallResultFunction",
handler="get_call_result.lambda_handler",
memory_size=512,
**global_python_function_arguments)
get_call_result_function.add_environment(
key="CallResultDynamoDBTable", value=call_result_table.table_name)
get_call_result_function.add_environment(key="S3Bucket",
value=file_bucket.bucket_name)
call_result_table.grant_read_data(grantee=get_call_result_function)
file_bucket.grant_read_write(get_call_result_function)
iterator_function = _lambda.Function(
self,
"IteratorFunction",
handler="iterator.lambda_handler",
**global_python_function_arguments)
iterator_function.add_permission(
id="IteratorFunctionLambdaInvokePermission",
principal=_iam.ServicePrincipal(service="connect.amazonaws.com"),
action="lambda:InvokeFunction",
source_account=self.account,
source_arn=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}"
)
create_appsync_call_task_function = _lambda.Function(
self,
"CreateAppSyncCallTaskFunction",
handler="create_appsync_call_task.lambda_handler",
**global_nodeje_function_arguments)
create_appsync_call_task_function.add_environment(
key="CallSqsQueueUrl", value=call_sqs_queue.queue_url)
create_appsync_call_task_function.add_environment(
key="CallRecordTableName", value=callout_record_table.table_name)
call_sqs_queue.grant_send_messages(create_appsync_call_task_function)
callout_record_table.grant_write_data(
create_appsync_call_task_function)
create_call_report_record_function = _lambda.Function(
self,
"CreateCallReportRecordFunction",
handler="create_call_report_record.lambda_handler",
**global_nodeje_function_arguments)
create_excel_call_task_function = _lambda.Function(
self,
"CreateExcelCallTaskFunction",
handler="create_excel_call_task.lambda_handler",
**global_python_function_arguments)
create_excel_call_task_function.add_environment(
key="CallSqsQueueUrl", value=call_sqs_queue.queue_url)
call_sqs_queue.grant_send_messages(create_excel_call_task_function)
create_excel_call_task_function.add_event_source(
source=_les.S3EventSource(bucket=file_bucket,
events=[_s3.EventType.OBJECT_CREATED],
filters=[
_s3.NotificationKeyFilter(
prefix="call_task",
suffix=".xlsx")
]))
start_callout_flow_function = _lambda.Function(
self,
"StartCalloutFlowFunction",
handler="start_call_out_flow.lambda_handler",
reserved_concurrent_executions=1,
**global_python_function_arguments)
start_callout_flow_function.add_environment(
key="CallSqsQueueUrl", value=call_sqs_queue.queue_url)
start_callout_flow_function.add_environment(
key="ResponseHandlerFunctionArn",
value=response_handler_function.function_arn)
start_callout_flow_function.add_environment(
key="IteratorFunctionArn", value=iterator_function.function_arn)
start_callout_flow_function.add_environment(
key="SendTaskSuccessFunctionArn",
value=send_task_success_function.function_arn)
start_callout_flow_function.add_environment(
key="S3Bucket", value=file_bucket.bucket_name)
start_callout_flow_function.add_event_source(
source=_les.SqsEventSource(queue=call_sqs_queue, batch_size=1))
file_bucket.grant_read_write(start_callout_flow_function)
call_state_machine_definition = {
"Comment":
"Reading messages from an SQS queue and iteratively processing each message.",
"StartAt": "Start",
"States": {
"Start": {
"Type": "Pass",
"Next": "Process Call Messages"
},
"Process Call Messages": {
"Type": "Map",
"Next": "Get Call Result",
"InputPath": "$",
"ItemsPath": "$",
"OutputPath": "$.[0]",
"Iterator": {
"StartAt": "Get Call out job",
"States": {
"Get Call out job": {
"Type": "Task",
"Resource":
get_callout_job_function.function_arn,
"Next": "Callout with AWS Connect"
},
"Callout with AWS Connect": {
"Type":
"Task",
"Resource":
"arn:aws:states:::sqs:sendMessage.waitForTaskToken",
"TimeoutSeconds":
timeout,
"Parameters": {
"QueueUrl": async_callout_queue.queue_url,
"MessageGroupId": "1",
"MessageBody": {
"Message.$": "$",
"TaskToken.$": "$$.Task.Token"
}
},
"Catch": [{
"ErrorEquals": ["States.Timeout"],
"ResultPath": None,
"Next": "Call Timeout"
}],
"Next":
"Save call result"
},
"Call Timeout": {
"Type": "Pass",
"ResultPath": None,
"Next": "Save call result"
},
"Save call result": {
"Type": "Task",
"Resource":
"arn:aws:states:::dynamodb:putItem",
"Parameters": {
"TableName": call_result_table.table_name,
"Item": {
"receiver_id": {
"S.$": "$.receiver_id"
},
"task_id": {
"S.$": "$.task_id"
},
"username": {
"S.$": "$.username"
},
"phone_number": {
"S.$": "$.phone_number"
},
"status": {
"S.$": "$.status"
},
"answers": {
"S.$": "$.answers"
},
"error": {
"S.$": "$.error"
},
"call_at": {
"S.$": "$.call_at"
}
}
},
"ResultPath": "$.Result",
"OutputPath": "$.task_id",
"End": True
}
}
}
},
"Get Call Result": {
"Type": "Task",
"Resource": get_call_result_function.function_arn,
"Next": "Create Call Report Record"
},
"Create Call Report Record": {
"Type": "Task",
"Resource":
create_call_report_record_function.function_arn,
"Next": "Send Completion message to SNS"
},
"Send Completion message to SNS": {
"Type": "Task",
"Resource": "arn:aws:states:::sns:publish",
"Parameters": {
"TopicArn": call_job_complete_sns_topic.topic_arn,
"Message.$": "$"
},
"Next": "Finish"
},
"Finish": {
"Type": "Succeed"
}
}
}
callout_state_machine_role = _iam.Role(
self,
"CalloutStatesExecutionRole",
assumed_by=_iam.ServicePrincipal(
f"states.{self.region}.amazonaws.com"))
callout_state_machine_role.add_to_policy(
_iam.PolicyStatement(
actions=[
"sqs:SendMessage", "dynamodb:PutItem",
"lambda:InvokeFunction", "SNS:Publish"
],
resources=[
async_callout_queue.queue_arn, call_result_table.table_arn,
get_callout_job_function.function_arn,
get_call_result_function.function_arn,
call_job_complete_sns_topic.topic_arn,
create_appsync_call_task_function.function_arn,
create_call_report_record_function.function_arn
]))
callout_state_machine = _sfn.CfnStateMachine(
self,
"CalloutStateMachine",
role_arn=callout_state_machine_role.role_arn,
definition_string=json.dumps(call_state_machine_definition))
send_task_success_function.add_to_role_policy(
_iam.PolicyStatement(actions=["states:SendTaskSuccess"],
resources=[callout_state_machine.ref]))
start_callout_flow_function.add_environment(
key="CalloutStateMachineArn", value=callout_state_machine.ref)
start_callout_flow_function.add_to_role_policy(
_iam.PolicyStatement(effect=_iam.Effect.ALLOW,
resources=[callout_state_machine.ref],
actions=['states:StartExecution']))
user_pool = _cognito.UserPool(
self, "UserPool", sign_in_type=_cognito.SignInType.USERNAME)
user_pool_client = _cognito.UserPoolClient(self,
"UserPoolClient",
user_pool=user_pool)
appsync_api = _appsync.GraphQLApi(
self,
"AppSyncApi",
name="AWSCalloutApi",
user_pool_config=_appsync.UserPoolConfig(
user_pool=user_pool,
default_action=_appsync.UserPoolDefaultAction.ALLOW),
log_config=_appsync.LogConfig(
field_log_level=_appsync.FieldLogLevel.ALL),
schema_definition_file=
f"{pathlib.Path(__file__).parent.absolute()}/schema.graphql")
callout_record_ddb_ds = appsync_api.add_dynamo_db_data_source(
name="CalloutRecordDdb",
description="Callout Record DynamoDB Data Source",
table=callout_record_table)
callout_record_ddb_ds.create_resolver(
type_name="Query",
field_name="getLatestCallTaskRecords",
request_mapping_template=_appsync.MappingTemplate.from_string(
'{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"TASK"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}'
),
response_mapping_template=_appsync.MappingTemplate.
dynamo_db_result_list())
callout_record_ddb_ds.create_resolver(
type_name="Query",
field_name="getLatestCallReportRecords",
request_mapping_template=_appsync.MappingTemplate.from_string(
'{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"REPORT"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}'
),
response_mapping_template=_appsync.MappingTemplate.
dynamo_db_result_list())
callout_record_ddb_ds.create_resolver(
type_name="Mutation",
field_name="createCallReport",
request_mapping_template=_appsync.MappingTemplate.from_string(
'{"version":"2017-02-28","operation":"PutItem","key":{"task_id":{"S":"${ctx.args.report.task_id}"},"created_at":{"N":"${ctx.args.report.created_at}"}},"attributeValues":$util.dynamodb.toMapValuesJson($ctx.args.report)}'
),
response_mapping_template=_appsync.MappingTemplate.
dynamo_db_result_item())
call_task_lambda_ds = appsync_api.add_lambda_data_source(
name="CallTaskLambda",
description="Call Task Lambda Data Source",
lambda_function=create_appsync_call_task_function)
call_task_lambda_ds.create_resolver(
type_name="Mutation",
field_name="createCallTask",
request_mapping_template=_appsync.MappingTemplate.lambda_request(
"$utils.toJson($ctx.args)"),
response_mapping_template=_appsync.MappingTemplate.lambda_result())
create_call_report_record_function.add_environment(
value=appsync_api.graph_ql_url, key="AppSyncGraphQlApiUrl")
create_call_report_record_function.add_to_role_policy(
statement=_iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=['appsync:GraphQL'],
resources=[
f"{appsync_api.arn}/types/Mutation/fields/createCallReport"
]))
core.CfnOutput(self,
id="OutputCallSqsQueue",
value=call_sqs_queue.queue_arn)
core.CfnOutput(self,
id="OutputCallJobCompletionSNSTopic",
value=call_job_complete_sns_topic.topic_arn)
core.CfnOutput(self,
id="OutputExcelFileS3Bucket",
value=file_bucket.bucket_name)
core.CfnOutput(self,
id="OutputStaticWebS3Bucket",
value=web_bucket.bucket_name)
core.CfnOutput(self,
id="OutputStaticWebUrl",
value=web_bucket.bucket_website_url)
identity_pool = _cognito.CfnIdentityPool(
self,
"IdentityPool",
allow_unauthenticated_identities=True,
cognito_identity_providers=[
_cognito.CfnIdentityPool.CognitoIdentityProviderProperty(
provider_name=user_pool.user_pool_provider_name,
client_id=user_pool_client.user_pool_client_id)
])
identity_pool_unauthorized_role = _iam.Role(
self,
'IdentityPoolUnAuthorizedRole',
assumed_by=_iam.FederatedPrincipal(
federated="cognito-identity.amazonaws.com",
assume_role_action="sts:AssumeRoleWithWebIdentity",
conditions={
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "unauthenticated"
}
}))
identity_pool_unauthorized_role.add_to_policy(
_iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=["appsync:GraphQL"],
resources=[
f"{appsync_api.arn}/types/*",
# f"{appsync_api.arn}/types/Query/fields/getLatestCallTaskRecords",
# f"{appsync_api.arn}/types/Query/fields/getLatestCallReportRecords",
# f"{appsync_api.arn}/types/Mutation/fields/createCallRecord",
# f"{appsync_api.arn}/types/Subscription/fields/createCallTask",
# f"{appsync_api.arn}/types/Subscription/fields/createCallReport"
]))
_cognito.CfnIdentityPoolRoleAttachment(
self,
"CognitoIdentityPoolRoleAttachment",
identity_pool_id=identity_pool.ref,
roles={
"unauthenticated": identity_pool_unauthorized_role.role_arn
})
core.CfnOutput(self, id="UserPoolId", value=user_pool.user_pool_id)
core.CfnOutput(self,
id="UserPoolClientId",
value=user_pool_client.user_pool_client_id)
core.CfnOutput(self, id="IdentityPoolId", value=identity_pool.ref)
def __init__(self, scope: core.Stack, id: str, **kwargs):
super().__init__(scope, id, **kwargs)
self.poc_config = {'api_poc': dict()}
self.read_config()
# shared stuff
self._vpc = ec2.Vpc(
self,
'api_poc-vpc',
cidr='10.0.0.0/23',
max_azs=1,
nat_gateways=1,
)
self._private_subnet_selection = self._vpc.select_subnets(
subnet_type=ec2.SubnetType.PRIVATE)
self._security_group = ec2.SecurityGroup.from_security_group_id(
self,
'default_sg',
security_group_id=self._vpc.vpc_default_security_group)
self._security_group.add_ingress_rule(description='redis',
peer=self._security_group,
connection=ec2.Port.tcp_range(
start_port=6379,
end_port=6379))
self._python3_lib_layer = _lambda.LayerVersion(
self,
'python3-lib-layer',
description="python3 module dependencies",
compatible_runtimes=[
_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_6
],
code=_lambda.Code.from_asset('layers/python3-lib-layer.zip'))
# redis cache cluster
self._cache_subnet_group = elasticache.CfnSubnetGroup(
self,
'cache_subnet_group',
description='elasticache subnet group',
subnet_ids=self._private_subnet_selection.subnet_ids,
cache_subnet_group_name='cache-subnet-group')
self._redis_cache = elasticache.CfnCacheCluster(
self,
'cache',
cache_node_type='cache.t2.micro',
num_cache_nodes=1,
engine='redis',
cache_subnet_group_name='cache-subnet-group',
vpc_security_group_ids=[self._security_group.security_group_id],
)
self._redis_cache.add_depends_on(self._cache_subnet_group)
# external API simulator lambda
api_handler = _lambda.Function(
self,
"external-api",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='external_api.handler',
layers=[self._python3_lib_layer],
vpc=self._vpc,
vpc_subnets=self._private_subnet_selection,
security_group=self._security_group,
log_retention=logs.RetentionDays.FIVE_DAYS,
tracing=_lambda.Tracing.ACTIVE)
api_handler.add_environment('REDIS_ADDRESS', self.redis_address)
api_handler.add_environment('REDIS_PORT', self.redis_port)
# API Gateway frontend to simulator lambda
self._api_gateway = apigw.LambdaRestApi(
self,
'external_api',
description='external API emulator',
options=apigw.StageOptions(stage_name='dev'),
handler=api_handler,
proxy=True)
job_dlq = sqs.Queue(self, 'job-dlq')
job_queue = sqs.Queue(self,
'job-queue',
dead_letter_queue=sqs.DeadLetterQueue(
queue=job_dlq, max_receive_count=3))
throttle_event_topic = sns.Topic(self, 'throttle-events-topic')
self.add_sns_subscriptions(throttle_event_topic)
worker = _lambda.Function(self,
'worker',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='worker.handler',
layers=[self._python3_lib_layer],
reserved_concurrent_executions=20,
timeout=core.Duration.minutes(1),
vpc=self._vpc,
vpc_subnets=self._private_subnet_selection,
security_group=self._security_group,
log_retention=logs.RetentionDays.FIVE_DAYS,
tracing=_lambda.Tracing.ACTIVE,
dead_letter_queue_enabled=False)
worker.add_environment('API_KEY', '212221848ab214821de993a9d')
worker.add_environment('JOB_QUEUE_URL', job_queue.queue_url)
worker.add_environment('THROTTLE_EVENTS_TOPIC',
throttle_event_topic.topic_arn)
worker.add_environment('REDIS_ADDRESS', self.redis_address)
worker.add_environment('REDIS_PORT', self.redis_port)
job_queue.grant_send_messages(worker)
throttle_event_topic.grant_publish(worker)
orchestrator = _lambda.Function(
self,
'orchestrator',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='orchestrator.handler',
layers=[self._python3_lib_layer],
reserved_concurrent_executions=1,
timeout=core.Duration.minutes(2),
vpc=self._vpc,
vpc_subnets=self._private_subnet_selection,
security_group=self._security_group,
log_retention=logs.RetentionDays.FIVE_DAYS,
tracing=_lambda.Tracing.ACTIVE,
)
orchestrator.add_environment('API_HOST_URL', self._api_gateway.url)
orchestrator.add_environment('JOB_QUEUE_URL', job_queue.queue_url)
orchestrator.add_environment('JOB_DLQ_URL', job_dlq.queue_url)
orchestrator.add_environment('THROTTLE_EVENTS_TOPIC',
throttle_event_topic.topic_arn)
orchestrator.add_environment('REDIS_ADDRESS', self.redis_address)
orchestrator.add_environment('REDIS_PORT', self.redis_port)
orchestrator.add_environment('WORKER_FUNCTION_ARN',
worker.function_arn)
job_queue.grant_consume_messages(orchestrator)
job_dlq.grant_send_messages(orchestrator)
throttle_event_topic.grant_publish(orchestrator)
worker.grant_invoke(orchestrator)
task_master = _lambda.Function(
self,
'task_master',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='task_master.handler',
layers=[self._python3_lib_layer],
reserved_concurrent_executions=1,
vpc=self._vpc,
vpc_subnets=self._private_subnet_selection,
security_group=self._security_group,
log_retention=logs.RetentionDays.FIVE_DAYS,
tracing=_lambda.Tracing.ACTIVE,
)
task_master.add_environment('SQS_URL', job_queue.queue_url)
task_master.add_environment('REDIS_ADDRESS', self.redis_address)
task_master.add_environment('REDIS_PORT', self.redis_port)
task_master.add_environment('API_HOST_URL', self._api_gateway.url)
job_queue.grant_send_messages(task_master)
slack_notify = _lambda.Function(
self,
'slack-notify',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset('lambda'),
handler='slack_notify.lambda_handler',
log_retention=logs.RetentionDays.FIVE_DAYS,
tracing=_lambda.Tracing.ACTIVE,
)
# lambda uses ssm parameter store to retrieve values
slack_notify.add_environment('encryptedHookUrlKey',
'/api_poc/notify/slack/hook_url')
slack_notify.add_environment('slackChannelKey',
'/api_poc/notify/slack/channel')
slack_notify.add_environment('notifySlack', 'false')
slack_notify.add_event_source(
event_sources.SnsEventSource(throttle_event_topic))
slack_notify.add_to_role_policy(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
# TODO fix least privilege
# actions=['ssm:GetParameter'],
# resources=['arn:aws:ssm:::parameter/api_poc/notify/slack/*'],
actions=['ssm:*'],
resources=['*'],
))
# kick off lambda(s) once per interval
rule = events.Rule(self,
'orchestrator_rule',
schedule=events.Schedule.rate(
core.Duration.hours(1)))
# See https://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html
rule.add_target(targets.LambdaFunction(orchestrator))
rule.add_target(targets.LambdaFunction(task_master))
# stack outputs
core.CfnOutput(self,
'Redis_Address',
value=self._redis_cache.attr_redis_endpoint_address +
':' + self._redis_cache.attr_redis_endpoint_port)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
###########################################################################
# AWS LAMBDA FUNCTIONS
###########################################################################
parse_image_list_file = aws_lambda.Function(
self,
'parse_image_list_file',
handler='parse_image_list_file.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('parse_image_list_file'),
memory_size=10240,
timeout=core.Duration.seconds(300),
log_retention=aws_logs.RetentionDays.ONE_DAY)
list_objects = aws_lambda.Function(
self,
'list_objects',
handler='list_objects.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('list_objects'),
memory_size=4096,
timeout=core.Duration.seconds(300),
log_retention=aws_logs.RetentionDays.ONE_DAY)
get_size_and_store = aws_lambda.Function(
self,
'get_size_and_store',
handler='get_size_and_store.lambda_handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=aws_lambda.Code.asset('get_size_and_store'),
memory_size=4096,
timeout=core.Duration.seconds(300),
log_retention=aws_logs.RetentionDays.ONE_DAY)
###########################################################################
# AMAZON S3 BUCKETS
###########################################################################
images_bucket = aws_s3.Bucket(self, "images_bucket")
###########################################################################
# LAMBDA SUPPLEMENTAL POLICIES
###########################################################################
lambda_supplemental_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=["s3:Get*", "s3:Head*", "s3:List*", "sqs:*", "es:*"],
resources=["*"])
parse_image_list_file.add_to_role_policy(
lambda_supplemental_policy_statement)
list_objects.add_to_role_policy(lambda_supplemental_policy_statement)
get_size_and_store.add_to_role_policy(
lambda_supplemental_policy_statement)
###########################################################################
# AWS SNS TOPICS
###########################################################################
# notification_topic = aws_sns.Topic(self, "notification_topic")
###########################################################################
# ADD AMAZON S3 BUCKET NOTIFICATIONS
###########################################################################
images_bucket.add_event_notification(
aws_s3.EventType.OBJECT_CREATED,
aws_s3_notifications.LambdaDestination(parse_image_list_file))
###########################################################################
# AWS SQS QUEUES
###########################################################################
comprehend_queue_iqueue = aws_sqs.Queue(self,
"comprehend_queue_iqueue")
comprehend_queue_iqueue_dlq = aws_sqs.DeadLetterQueue(
max_receive_count=10, queue=comprehend_queue_iqueue)
comprehend_queue = aws_sqs.Queue(
self,
"comprehend_queue",
visibility_timeout=core.Duration.seconds(301),
dead_letter_queue=comprehend_queue_iqueue_dlq)
rekognition_queue_iqueue = aws_sqs.Queue(self,
"rekognition_queue_iqueue")
rekognition_queue_dlq = aws_sqs.DeadLetterQueue(
max_receive_count=10, queue=rekognition_queue_iqueue)
rekognition_queue = aws_sqs.Queue(
self,
"rekognition_queue",
visibility_timeout=core.Duration.seconds(301),
dead_letter_queue=rekognition_queue_dlq)
object_queue_iqueue = aws_sqs.Queue(self, "object_queue_iqueue")
object_queue_dlq = aws_sqs.DeadLetterQueue(max_receive_count=10,
queue=object_queue_iqueue)
object_queue = aws_sqs.Queue(
self,
"object_queue",
visibility_timeout=core.Duration.seconds(301),
dead_letter_queue=object_queue_dlq)
###########################################################################
# AWS LAMBDA SQS EVENT SOURCE
###########################################################################
get_size_and_store.add_event_source(
SqsEventSource(object_queue, batch_size=10))
###########################################################################
# AWS ELASTICSEARCH DOMAIN
###########################################################################
s3workflow_domain = aws_elasticsearch.Domain(
self,
"s3workflow_domain",
version=aws_elasticsearch.ElasticsearchVersion.V7_1,
capacity={
"master_nodes": 3,
"data_nodes": 4
},
ebs={"volume_size": 100},
zone_awareness={"availability_zone_count": 2},
logging={
"slow_search_log_enabled": True,
"app_log_enabled": True,
"slow_index_log_enabled": True
})
###########################################################################
# AMAZON COGNITO USER POOL
###########################################################################
s3workflow_pool = aws_cognito.UserPool(
self,
"s3workflow-pool",
account_recovery=None,
auto_verify=None,
custom_attributes=None,
email_settings=None,
enable_sms_role=None,
lambda_triggers=None,
mfa=None,
mfa_second_factor=None,
password_policy=None,
self_sign_up_enabled=None,
sign_in_aliases=aws_cognito.SignInAliases(email=True,
phone=None,
preferred_username=None,
username=True),
sign_in_case_sensitive=None,
sms_role=None,
sms_role_external_id=None,
standard_attributes=None,
user_invitation=None,
user_pool_name=None,
user_verification=None)
###########################################################################
# AMAZON VPC
###########################################################################
vpc = aws_ec2.Vpc(self, "s3workflowVPC",
max_azs=3) # default is all AZs in region
###########################################################################
# AMAZON ECS CLUSTER
###########################################################################
cluster = aws_ecs.Cluster(self, "s3", vpc=vpc)
###########################################################################
# AMAZON ECS Repositories
###########################################################################
rekognition_repository = aws_ecr.Repository(
self,
"rekognition_repository",
image_scan_on_push=True,
removal_policy=core.RemovalPolicy("DESTROY"))
comprehend_repository = aws_ecr.Repository(
self,
"comprehend_repository",
image_scan_on_push=True,
removal_policy=core.RemovalPolicy("DESTROY"))
###########################################################################
# AMAZON ECS Roles and Policies
###########################################################################
task_execution_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"logs:*", "ecs:*", "ec2:*", "elasticloadbalancing:*", "ecr:*"
],
resources=["*"])
task_execution_policy_document = aws_iam.PolicyDocument()
task_execution_policy_document.add_statements(
task_execution_policy_statement)
task_execution_policy = aws_iam.Policy(
self,
"task_execution_policy",
document=task_execution_policy_document)
task_execution_role = aws_iam.Role(
self,
"task_execution_role",
assumed_by=aws_iam.ServicePrincipal('ecs-tasks.amazonaws.com'))
task_execution_role.attach_inline_policy(task_execution_policy)
task_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
actions=[
"logs:*", "xray:*", "sqs:*", "s3:*", "rekognition:*",
"comprehend:*", "es:*"
],
resources=["*"])
task_policy_document = aws_iam.PolicyDocument()
task_policy_document.add_statements(task_policy_statement)
task_policy = aws_iam.Policy(self,
"task_policy",
document=task_policy_document)
task_role = aws_iam.Role(
self,
"task_role",
assumed_by=aws_iam.ServicePrincipal('ecs-tasks.amazonaws.com'))
task_role.attach_inline_policy(task_policy)
###########################################################################
# AMAZON ECS Task definitions
###########################################################################
rekognition_task_definition = aws_ecs.TaskDefinition(
self,
"rekognition_task_definition",
compatibility=aws_ecs.Compatibility("FARGATE"),
cpu="1024",
# ipc_mode=None,
memory_mib="2048",
network_mode=aws_ecs.NetworkMode("AWS_VPC"),
# pid_mode=None, #Not supported in Fargate and Windows containers
# placement_constraints=None,
execution_role=task_execution_role,
# family=None,
# proxy_configuration=None,
task_role=task_role
# volumes=None
)
comprehend_task_definition = aws_ecs.TaskDefinition(
self,
"comprehend_task_definition",
compatibility=aws_ecs.Compatibility("FARGATE"),
cpu="1024",
# ipc_mode=None,
memory_mib="2048",
network_mode=aws_ecs.NetworkMode("AWS_VPC"),
# pid_mode=None, #Not supported in Fargate and Windows containers
# placement_constraints=None,
execution_role=task_execution_role,
# family=None,
# proxy_configuration=None,
task_role=task_role
# volumes=None
)
###########################################################################
# AMAZON ECS Images
###########################################################################
rekognition_ecr_image = aws_ecs.EcrImage(
repository=rekognition_repository, tag="latest")
comprehend_ecr_image = aws_ecs.EcrImage(
repository=comprehend_repository, tag="latest")
###########################################################################
# ENVIRONMENT VARIABLES
###########################################################################
environment_variables = {}
environment_variables["COMPREHEND_QUEUE"] = comprehend_queue.queue_url
environment_variables[
"REKOGNITION_QUEUE"] = rekognition_queue.queue_url
environment_variables["IMAGES_BUCKET"] = images_bucket.bucket_name
environment_variables[
"ELASTICSEARCH_HOST"] = s3workflow_domain.domain_endpoint
parse_image_list_file.add_environment(
"ELASTICSEARCH_HOST", s3workflow_domain.domain_endpoint)
parse_image_list_file.add_environment("QUEUEURL",
rekognition_queue.queue_url)
parse_image_list_file.add_environment("DEBUG", "False")
parse_image_list_file.add_environment("BUCKET", "-")
parse_image_list_file.add_environment("KEY", "-")
list_objects.add_environment("QUEUEURL", object_queue.queue_url)
list_objects.add_environment("ELASTICSEARCH_HOST",
s3workflow_domain.domain_endpoint)
list_objects.add_environment("S3_BUCKET_NAME",
images_bucket.bucket_name)
list_objects.add_environment("S3_BUCKET_PREFIX", "images/")
list_objects.add_environment("S3_BUCKET_SUFFIX", "")
list_objects.add_environment("LOGGING_LEVEL", "INFO")
get_size_and_store.add_environment("QUEUEURL", object_queue.queue_url)
get_size_and_store.add_environment("ELASTICSEARCH_HOST",
s3workflow_domain.domain_endpoint)
get_size_and_store.add_environment("S3_BUCKET_NAME",
images_bucket.bucket_name)
get_size_and_store.add_environment("S3_BUCKET_PREFIX", "images/")
get_size_and_store.add_environment("S3_BUCKET_SUFFIX", "")
get_size_and_store.add_environment("LOGGING_LEVEL", "INFO")
###########################################################################
# ECS Log Drivers
###########################################################################
rekognition_task_log_driver = aws_ecs.LogDriver.aws_logs(
stream_prefix="s3workflow",
log_retention=aws_logs.RetentionDays("ONE_DAY"))
comprehend_task_log_driver = aws_ecs.LogDriver.aws_logs(
stream_prefix="s3workflow",
log_retention=aws_logs.RetentionDays("ONE_DAY"))
###########################################################################
# ECS Task Definitions
###########################################################################
rekognition_task_definition.add_container(
"rekognition_task_definition",
image=rekognition_ecr_image,
memory_reservation_mib=1024,
environment=environment_variables,
logging=rekognition_task_log_driver)
comprehend_task_definition.add_container(
"comprehend_task_definition",
image=comprehend_ecr_image,
memory_reservation_mib=1024,
environment=environment_variables,
logging=comprehend_task_log_driver)
###########################################################################
# AWS ROUTE53 HOSTED ZONE
###########################################################################
hosted_zone = aws_route53.HostedZone(
self,
"hosted_zone",
zone_name="s3workflow.com",
comment="private hosted zone for s3workflow system")
hosted_zone.add_vpc(vpc)
def __init__(
self,
scope: Construct,
stack_id: str,
*,
botocore_lambda_layer: aws_lambda_python.PythonLayerVersion,
env_name: str,
storage_bucket: aws_s3.Bucket,
validation_results_table: Table,
) -> None:
# pylint: disable=too-many-locals, too-many-statements
super().__init__(scope, stack_id)
############################################################################################
# PROCESSING ASSETS TABLE
processing_assets_table = Table(
self,
f"{env_name}-processing-assets",
env_name=env_name,
parameter_name=ParameterName.PROCESSING_ASSETS_TABLE_NAME,
sort_key=aws_dynamodb.Attribute(name="sk", type=aws_dynamodb.AttributeType.STRING),
)
############################################################################################
# BATCH JOB DEPENDENCIES
batch_job_queue = BatchJobQueue(
self,
"batch-job-queue",
env_name=env_name,
processing_assets_table=processing_assets_table,
).job_queue
s3_read_only_access_policy = aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3ReadOnlyAccess"
)
############################################################################################
# UPDATE CATALOG UPDATE MESSAGE QUEUE
dead_letter_queue = aws_sqs.Queue(
self,
"dead-letter-queue",
visibility_timeout=LAMBDA_TIMEOUT,
)
self.message_queue = aws_sqs.Queue(
self,
"update-catalog-message-queue",
visibility_timeout=LAMBDA_TIMEOUT,
dead_letter_queue=aws_sqs.DeadLetterQueue(max_receive_count=3, queue=dead_letter_queue),
)
self.message_queue_name_parameter = aws_ssm.StringParameter(
self,
"update-catalog-message-queue-name",
string_value=self.message_queue.queue_name,
description=f"Update Catalog Message Queue Name for {env_name}",
parameter_name=ParameterName.UPDATE_CATALOG_MESSAGE_QUEUE_NAME.value,
)
populate_catalog_lambda = BundledLambdaFunction(
self,
"populate-catalog-bundled-lambda-function",
directory="populate_catalog",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
botocore_lambda_layer=botocore_lambda_layer,
)
self.message_queue.grant_consume_messages(populate_catalog_lambda)
populate_catalog_lambda.add_event_source(
SqsEventSource(self.message_queue, batch_size=1) # type: ignore[arg-type]
)
############################################################################################
# STATE MACHINE TASKS
check_stac_metadata_task = LambdaTask(
self,
"check-stac-metadata-task",
directory="check_stac_metadata",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
assert check_stac_metadata_task.lambda_function.role
check_stac_metadata_task.lambda_function.role.add_managed_policy(
policy=s3_read_only_access_policy
)
for table in [processing_assets_table, validation_results_table]:
table.grant_read_write_data(check_stac_metadata_task.lambda_function)
table.grant(
check_stac_metadata_task.lambda_function,
"dynamodb:DescribeTable",
)
content_iterator_task = LambdaTask(
self,
"content-iterator-task",
directory="content_iterator",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{CONTENT_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
check_files_checksums_directory = "check_files_checksums"
check_files_checksums_default_payload_object = {
f"{DATASET_ID_KEY}.$": f"$.{DATASET_ID_KEY}",
f"{VERSION_ID_KEY}.$": f"$.{VERSION_ID_KEY}",
f"{METADATA_URL_KEY}.$": f"$.{METADATA_URL_KEY}",
f"{FIRST_ITEM_KEY}.$": f"$.{CONTENT_KEY}.{FIRST_ITEM_KEY}",
f"{ASSETS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{ASSETS_TABLE_NAME_KEY}",
f"{RESULTS_TABLE_NAME_KEY}.$": f"$.{CONTENT_KEY}.{RESULTS_TABLE_NAME_KEY}",
}
check_files_checksums_single_task = BatchSubmitJobTask(
self,
"check-files-checksums-single-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
)
array_size = int(
aws_stepfunctions.JsonPath.number_at(f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}")
)
check_files_checksums_array_task = BatchSubmitJobTask(
self,
"check-files-checksums-array-task",
env_name=env_name,
directory=check_files_checksums_directory,
s3_policy=s3_read_only_access_policy,
job_queue=batch_job_queue,
payload_object=check_files_checksums_default_payload_object,
container_overrides_command=[
"--dataset-id",
f"Ref::{DATASET_ID_KEY}",
"--version-id",
f"Ref::{VERSION_ID_KEY}",
"--first-item",
f"Ref::{FIRST_ITEM_KEY}",
"--assets-table-name",
f"Ref::{ASSETS_TABLE_NAME_KEY}",
"--results-table-name",
f"Ref::{RESULTS_TABLE_NAME_KEY}",
],
array_size=array_size,
)
for reader in [
content_iterator_task.lambda_function,
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
processing_assets_table.grant_read_data(reader) # type: ignore[arg-type]
processing_assets_table.grant(
reader, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
for writer in [
check_files_checksums_single_task.job_role,
check_files_checksums_array_task.job_role,
]:
validation_results_table.grant_read_write_data(writer) # type: ignore[arg-type]
validation_results_table.grant(
writer, "dynamodb:DescribeTable" # type: ignore[arg-type]
)
validation_summary_task = LambdaTask(
self,
"validation-summary-task",
directory="validation_summary",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{VALIDATION_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(validation_summary_task.lambda_function)
validation_results_table.grant(
validation_summary_task.lambda_function, "dynamodb:DescribeTable"
)
import_dataset_role = aws_iam.Role(
self,
"import-dataset",
assumed_by=aws_iam.ServicePrincipal( # type: ignore[arg-type]
"batchoperations.s3.amazonaws.com"
),
)
import_asset_file_function = ImportFileFunction(
self,
directory="import_asset_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_metadata_file_function = ImportFileFunction(
self,
directory="import_metadata_file",
invoker=import_dataset_role,
env_name=env_name,
botocore_lambda_layer=botocore_lambda_layer,
)
import_dataset_task = LambdaTask(
self,
"import-dataset-task",
directory="import_dataset",
botocore_lambda_layer=botocore_lambda_layer,
result_path=f"$.{IMPORT_DATASET_KEY}",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(
resources=[import_dataset_role.role_arn],
actions=["iam:PassRole"],
),
)
import_dataset_task.lambda_function.add_to_role_policy(
aws_iam.PolicyStatement(resources=["*"], actions=["s3:CreateJob"])
)
for table in [processing_assets_table]:
table.grant_read_data(import_dataset_task.lambda_function)
table.grant(import_dataset_task.lambda_function, "dynamodb:DescribeTable")
# Import status check
wait_before_upload_status_check = Wait(
self,
"wait-before-upload-status-check",
time=WaitTime.duration(Duration.seconds(10)),
)
upload_status_task = LambdaTask(
self,
"upload-status",
directory="upload_status",
botocore_lambda_layer=botocore_lambda_layer,
result_path="$.upload_status",
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
validation_results_table.grant_read_data(upload_status_task.lambda_function)
validation_results_table.grant(upload_status_task.lambda_function, "dynamodb:DescribeTable")
upload_status_task.lambda_function.add_to_role_policy(ALLOW_DESCRIBE_ANY_S3_JOB)
# Parameters
import_asset_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import asset file function arn",
string_value=import_asset_file_function.function_arn,
description=f"Import asset file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_ASSET_FILE_FUNCTION_TASK_ARN.value,
)
import_metadata_file_function_arn_parameter = aws_ssm.StringParameter(
self,
"import metadata file function arn",
string_value=import_metadata_file_function.function_arn,
description=f"Import metadata file function ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_METADATA_FILE_FUNCTION_TASK_ARN.value,
)
import_dataset_role_arn_parameter = aws_ssm.StringParameter(
self,
"import dataset role arn",
string_value=import_dataset_role.role_arn,
description=f"Import dataset role ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_IMPORT_DATASET_ROLE_ARN.value,
)
update_dataset_catalog = LambdaTask(
self,
"update-dataset-catalog",
directory="update_dataset_catalog",
botocore_lambda_layer=botocore_lambda_layer,
extra_environment={ENV_NAME_VARIABLE_NAME: env_name},
)
self.message_queue.grant_send_messages(update_dataset_catalog.lambda_function)
for storage_writer in [
import_dataset_role,
import_dataset_task.lambda_function,
import_asset_file_function,
import_metadata_file_function,
populate_catalog_lambda,
update_dataset_catalog.lambda_function,
]:
storage_bucket.grant_read_write(storage_writer) # type: ignore[arg-type]
grant_parameter_read_access(
{
import_asset_file_function_arn_parameter: [import_dataset_task.lambda_function],
import_dataset_role_arn_parameter: [import_dataset_task.lambda_function],
import_metadata_file_function_arn_parameter: [import_dataset_task.lambda_function],
processing_assets_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
import_dataset_task.lambda_function,
],
validation_results_table.name_parameter: [
check_stac_metadata_task.lambda_function,
content_iterator_task.lambda_function,
validation_summary_task.lambda_function,
upload_status_task.lambda_function,
],
self.message_queue_name_parameter: [update_dataset_catalog.lambda_function],
}
)
success_task = aws_stepfunctions.Succeed(self, "success")
upload_failure = aws_stepfunctions.Fail(self, "upload failure")
validation_failure = aws_stepfunctions.Succeed(self, "validation failure")
############################################################################################
# STATE MACHINE
dataset_version_creation_definition = (
check_stac_metadata_task.next(content_iterator_task)
.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "check_files_checksums_maybe_array"
)
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{ITERATION_SIZE_KEY}", 1
),
check_files_checksums_single_task.batch_submit_job,
)
.otherwise(check_files_checksums_array_task.batch_submit_job)
.afterwards()
)
.next(
aws_stepfunctions.Choice(self, "content_iteration_finished")
.when(
aws_stepfunctions.Condition.number_equals(
f"$.{CONTENT_KEY}.{NEXT_ITEM_KEY}", -1
),
validation_summary_task.next(
aws_stepfunctions.Choice( # type: ignore[arg-type]
self, "validation_successful"
)
.when(
aws_stepfunctions.Condition.boolean_equals(
f"$.{VALIDATION_KEY}.{SUCCESS_KEY}", True
),
import_dataset_task.next(
wait_before_upload_status_check # type: ignore[arg-type]
)
.next(upload_status_task)
.next(
aws_stepfunctions.Choice(
self, "import_completed" # type: ignore[arg-type]
)
.when(
aws_stepfunctions.Condition.and_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Complete"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Complete",
),
),
update_dataset_catalog.next(
success_task # type: ignore[arg-type]
),
)
.when(
aws_stepfunctions.Condition.or_(
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{ASSET_UPLOAD_KEY}.status", "Failed"
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Cancelled",
),
aws_stepfunctions.Condition.string_equals(
f"$.upload_status.{METADATA_UPLOAD_KEY}.status",
"Failed",
),
),
upload_failure, # type: ignore[arg-type]
)
.otherwise(
wait_before_upload_status_check # type: ignore[arg-type]
)
),
)
.otherwise(validation_failure) # type: ignore[arg-type]
),
)
.otherwise(content_iterator_task)
)
)
self.state_machine = aws_stepfunctions.StateMachine(
self,
f"{env_name}-dataset-version-creation",
definition=dataset_version_creation_definition, # type: ignore[arg-type]
)
self.state_machine_parameter = aws_ssm.StringParameter(
self,
"state machine arn",
description=f"State machine ARN for {env_name}",
parameter_name=ParameterName.PROCESSING_DATASET_VERSION_CREATION_STEP_FUNCTION_ARN.value, # pylint:disable=line-too-long
string_value=self.state_machine.state_machine_arn,
)
Tags.of(self).add("ApplicationLayer", "processing") # type: ignore[arg-type]
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id)
lambda_dir = kwargs["lambda_dir"]
# Note: A dead-letter queue is optional but it helps capture any failed messages
dlq = sqs.Queue(self,
id="dead_letter_queue_id",
retention_period=Duration.days(7))
dead_letter_queue = sqs.DeadLetterQueue(max_receive_count=1, queue=dlq)
upload_queue = sqs.Queue(self,
id="sample_queue_id",
visibility_timeout=Duration.seconds(30),
dead_letter_queue=dead_letter_queue)
sqs_subscription = sns_subs.SqsSubscription(upload_queue,
raw_message_delivery=True)
upload_event_topic = sns.Topic(self, id="sample_sns_topic_id")
# This binds the SNS Topic to the SQS Queue
upload_event_topic.add_subscription(sqs_subscription)
# Note: Lifecycle Rules are optional but are included here to keep costs
# low by cleaning up old files or moving them to lower cost storage options
s3_bucket = s3.Bucket(
self,
id="sample_bucket_id",
block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
versioned=True,
lifecycle_rules=[
s3.LifecycleRule(
enabled=True,
expiration=Duration.days(365),
transitions=[
s3.Transition(
storage_class=s3.StorageClass.INFREQUENT_ACCESS,
transition_after=Duration.days(30)),
s3.Transition(storage_class=s3.StorageClass.GLACIER,
transition_after=Duration.days(90)),
])
])
# Note: If you don't specify a filter all uploads will trigger an event.
# Also, modifying the event type will handle other object operations
# This binds the S3 bucket to the SNS Topic
s3_bucket.add_event_notification(
s3.EventType.OBJECT_CREATED_PUT,
s3n.SnsDestination(upload_event_topic),
s3.NotificationKeyFilter(prefix="uploads", suffix=".csv"))
function = _lambda.Function(
self,
"lambda_function",
runtime=_lambda.Runtime.PYTHON_3_9,
handler="lambda_function.handler",
code=_lambda.Code.from_asset(path=lambda_dir))
# This binds the lambda to the SQS Queue
invoke_event_source = lambda_events.SqsEventSource(upload_queue)
function.add_event_source(invoke_event_source)
# Examples of CloudFormation outputs
CfnOutput(
self,
"UploadFileToS3Example",
value="aws s3 cp <local-path-to-file> s3://{}/".format(
s3_bucket.bucket_name),
description=
"Upload a file to S3 (using AWS CLI) to trigger the SQS chain",
)
CfnOutput(
self,
"UploadSqsQueueUrl",
value=upload_queue.queue_url,
description="Link to the SQS Queue triggered on S3 uploads",
)
CfnOutput(
self,
"LambdaFunctionName",
value=function.function_name,
)
CfnOutput(
self,
"LambdaFunctionLogGroupName",
value=function.log_group.log_group_name,
)
def __init__(self, scope: cdk.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Image Bucket
image_bucket = s3.Bucket(self,
IMG_BUCKET_NAME,
removal_policy=cdk.RemovalPolicy.DESTROY)
cdk.CfnOutput(self, "imageBucket", value=image_bucket.bucket_name)
image_bucket.add_cors_rule(
allowed_methods=[s3.HttpMethods.GET, s3.HttpMethods.PUT],
allowed_origins=["*"],
allowed_headers=["*"],
max_age=3000,
)
# Thumbnail Bucket
resized_image_bucket = s3.Bucket(
self,
RESIZED_IMG_BUCKET_NAME,
removal_policy=cdk.RemovalPolicy.DESTROY)
cdk.CfnOutput(self,
"resizedBucket",
value=resized_image_bucket.bucket_name)
resized_image_bucket.add_cors_rule(
allowed_methods=[s3.HttpMethods.GET, s3.HttpMethods.PUT],
allowed_origins=["*"],
allowed_headers=["*"],
max_age=3000,
)
# S3 Static bucket for website code
web_bucket = s3.Bucket(
self,
WEBSITE_BUCKET_NAME,
website_index_document="index.html",
website_error_document="index.html",
removal_policy=cdk.RemovalPolicy.DESTROY,
# uncomment this and delete the policy statement below to allow public access to our
# static website
# public_read_access=true
)
web_policy_statement = iam.PolicyStatement(
actions=["s3:GetObject"],
resources=[web_bucket.arn_for_objects("*")],
principals=[iam.AnyPrincipal()],
conditions={"IpAddress": {
"aws:SourceIp": ["139.138.203.36"]
}},
)
web_bucket.add_to_resource_policy(web_policy_statement)
cdk.CfnOutput(self,
"bucketURL",
value=web_bucket.bucket_website_domain_name)
# Deploy site contents to S3 Bucket
s3_dep.BucketDeployment(
self,
"DeployWebsite",
sources=[s3_dep.Source.asset("./public")],
destination_bucket=web_bucket,
)
# DynamoDB to store image labels
partition_key = dynamodb.Attribute(name="image",
type=dynamodb.AttributeType.STRING)
table = dynamodb.Table(
self,
"ImageLabels",
partition_key=partition_key,
removal_policy=cdk.RemovalPolicy.DESTROY,
)
cdk.CfnOutput(self, "ddbTable", value=table.table_name)
# Lambda layer for Pillow library
layer = lb.LayerVersion(
self,
"pil",
code=lb.Code.from_asset("reklayer"),
compatible_runtimes=[lb.Runtime.PYTHON_3_7],
license="Apache-2.0",
description=
"A layer to enable the PIL library in our Rekognition Lambda",
)
# Lambda function
rek_fn = lb.Function(
self,
"rekognitionFunction",
code=lb.Code.from_asset("rekognitionFunction"),
runtime=lb.Runtime.PYTHON_3_7,
handler="index.handler",
timeout=cdk.Duration.seconds(30),
memory_size=1024,
layers=[layer],
environment={
"TABLE": table.table_name,
"BUCKET": image_bucket.bucket_name,
"THUMBBUCKET": resized_image_bucket.bucket_name,
},
)
image_bucket.grant_read(rek_fn)
resized_image_bucket.grant_write(rek_fn)
table.grant_write_data(rek_fn)
rek_fn.add_to_role_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=["rekognition:DetectLabels"],
resources=["*"]))
# Lambda for Synchronous front end
serviceFn = lb.Function(
self,
"serviceFunction",
code=lb.Code.from_asset("servicelambda"),
runtime=lb.Runtime.PYTHON_3_7,
handler="index.handler",
environment={
"TABLE": table.table_name,
"BUCKET": image_bucket.bucket_name,
"RESIZEDBUCKET": resized_image_bucket.bucket_name,
},
)
image_bucket.grant_write(serviceFn)
resized_image_bucket.grant_write(serviceFn)
table.grant_read_write_data(serviceFn)
# Cognito User Pool Auth
auto_verified_attrs = cognito.AutoVerifiedAttrs(email=True)
sign_in_aliases = cognito.SignInAliases(email=True, username=True)
user_pool = cognito.UserPool(
self,
"UserPool",
self_sign_up_enabled=True,
auto_verify=auto_verified_attrs,
sign_in_aliases=sign_in_aliases,
)
user_pool_client = cognito.UserPoolClient(self,
"UserPoolClient",
user_pool=user_pool,
generate_secret=False)
identity_pool = cognito.CfnIdentityPool(
self,
"ImageRekognitionIdentityPool",
allow_unauthenticated_identities=False,
cognito_identity_providers=[{
"clientId":
user_pool_client.user_pool_client_id,
"providerName":
user_pool.user_pool_provider_name,
}],
)
# API Gateway
cors_options = apigw.CorsOptions(allow_origins=apigw.Cors.ALL_ORIGINS,
allow_methods=apigw.Cors.ALL_METHODS)
api = apigw.LambdaRestApi(
self,
"imageAPI",
default_cors_preflight_options=cors_options,
handler=serviceFn,
proxy=False,
)
auth = apigw.CfnAuthorizer(
self,
"ApiGatewayAuthorizer",
name="customer-authorizer",
identity_source="method.request.header.Authorization",
provider_arns=[user_pool.user_pool_arn],
rest_api_id=api.rest_api_id,
# type=apigw.AuthorizationType.COGNITO,
type="COGNITO_USER_POOLS",
)
assumed_by = iam.FederatedPrincipal(
"cognito-identity.amazon.com",
conditions={
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "authenticated"
},
},
assume_role_action="sts:AssumeRoleWithWebIdentity",
)
authenticated_role = iam.Role(
self,
"ImageRekognitionAuthenticatedRole",
assumed_by=assumed_by,
)
# IAM policy granting users permission to get and put their pictures
policy_statement = iam.PolicyStatement(
actions=["s3:GetObject", "s3:PutObject"],
effect=iam.Effect.ALLOW,
resources=[
image_bucket.bucket_arn +
"/private/${cognito-identity.amazonaws.com:sub}/*",
image_bucket.bucket_arn +
"/private/${cognito-identity.amazonaws.com:sub}/",
resized_image_bucket.bucket_arn +
"/private/${cognito-identity.amazonaws.com:sub}/*",
resized_image_bucket.bucket_arn +
"/private/${cognito-identity.amazonaws.com:sub}/",
],
)
# IAM policy granting users permission to list their pictures
list_policy_statement = iam.PolicyStatement(
actions=["s3:ListBucket"],
effect=iam.Effect.ALLOW,
resources=[
image_bucket.bucket_arn, resized_image_bucket.bucket_arn
],
conditions={
"StringLike": {
"s3:prefix":
["private/${cognito-identity.amazonaws.com:sub}/*"]
}
},
)
authenticated_role.add_to_policy(policy_statement)
authenticated_role.add_to_policy(list_policy_statement)
# Attach role to our Identity Pool
cognito.CfnIdentityPoolRoleAttachment(
self,
"IdentityPoolRoleAttachment",
identity_pool_id=identity_pool.ref,
roles={"authenticated": authenticated_role.role_arn},
)
# Get some outputs from cognito
cdk.CfnOutput(self, "UserPoolId", value=user_pool.user_pool_id)
cdk.CfnOutput(self,
"AppClientId",
value=user_pool_client.user_pool_client_id)
cdk.CfnOutput(self, "IdentityPoolId", value=identity_pool.ref)
# New Amazon API Gateway with AWS Lambda Integration
success_response = apigw.IntegrationResponse(
status_code="200",
response_parameters={
"method.response.header.Access-Control-Allow-Origin": "'*'"
},
)
error_response = apigw.IntegrationResponse(
selection_pattern="(\n|.)+",
status_code="500",
response_parameters={
"method.response.header.Access-Control-Allow-Origin": "'*'"
},
)
request_template = json.dumps({
"action":
"$util.escapeJavaScript($input.params('action'))",
"key":
"$util.escapeJavaScript($input.params('key'))",
})
lambda_integration = apigw.LambdaIntegration(
serviceFn,
proxy=False,
request_parameters={
"integration.request.querystring.action":
"method.request.querystring.action",
"integration.request.querystring.key":
"method.request.querystring.key",
},
request_templates={"application/json": request_template},
passthrough_behavior=apigw.PassthroughBehavior.WHEN_NO_TEMPLATES,
integration_responses=[success_response, error_response],
)
imageAPI = api.root.add_resource("images")
success_resp = apigw.MethodResponse(
status_code="200",
response_parameters={
"method.response.header.Access-Control-Allow-Origin": True
},
)
error_resp = apigw.MethodResponse(
status_code="500",
response_parameters={
"method.response.header.Access-Control-Allow-Origin": True
},
)
# GET /images
get_method = imageAPI.add_method(
"GET",
lambda_integration,
authorization_type=apigw.AuthorizationType.COGNITO,
request_parameters={
"method.request.querystring.action": True,
"method.request.querystring.key": True,
},
method_responses=[success_resp, error_resp],
)
# DELETE /images
delete_method = imageAPI.add_method(
"DELETE",
lambda_integration,
authorization_type=apigw.AuthorizationType.COGNITO,
request_parameters={
"method.request.querystring.action": True,
"method.request.querystring.key": True,
},
method_responses=[success_resp, error_resp],
)
# Override the authorizer id because it doesn't work when defininting it as a param
# in add_method
get_method_resource = get_method.node.find_child("Resource")
get_method_resource.add_property_override("AuthorizerId", auth.ref)
delete_method_resource = delete_method.node.find_child("Resource")
delete_method_resource.add_property_override("AuthorizerId", auth.ref)
# Building SQS queue and DeadLetter Queue
dl_queue = sqs.Queue(
self,
"ImageDLQueue",
queue_name="ImageDLQueue",
)
dl_queue_opts = sqs.DeadLetterQueue(max_receive_count=2,
queue=dl_queue)
queue = sqs.Queue(
self,
"ImageQueue",
queue_name="ImageQueue",
visibility_timeout=cdk.Duration.seconds(30),
receive_message_wait_time=cdk.Duration.seconds(20),
dead_letter_queue=dl_queue_opts,
)
# S3 Bucket Create Notification to SQS
# Whenever an image is uploaded add it to the queue
image_bucket.add_object_created_notification(
s3n.SqsDestination(queue),
s3.NotificationKeyFilter(prefix="private/"))
def create_all_queues(self) -> None:
"""
Create all STACK queues, attach subscriptions and alarms
"""
# General DLQs for lambdas (not API)
self.create_queue(id="dead_letter_queue")
general_dlq_alarm = cloudwatch.Alarm(
self,
"DLQAlarm",
metric=self.queues_["dead_letter_queue"].metric(
"ApproximateNumberOfMessagesVisible"),
evaluation_periods=1,
threshold=0.0,
comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD,
)
general_dlq_alarm.add_alarm_action(
cw_actions.SnsAction(self.topics_["alarm_topic"]))
# DLQ for API lambdas
self.create_queue(id="api_dead_letter_queue")
api_dlq_alarm = cloudwatch.Alarm(
self,
"APIDLQAlarm",
metric=self.queues_["api_dead_letter_queue"].metric(
"ApproximateNumberOfMessagesVisible"),
evaluation_periods=1,
threshold=0.0,
comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD,
)
api_dlq_alarm.add_alarm_action(
cw_actions.SnsAction(self.topics_["alarm_topic"]))
# The new_scenes_queue subscribe to CBERS 4/4A quicklooks notifications. The
# STAC items are generated from the original INPE metadata file as
# soon as the quicklooks are created in the PDS bucket
# This code fragment creates the queue, the associated dlq and
# subscribe to CBERS 4/4A quicklook notification topics
self.create_queue(
id="process_new_scenes_queue_dlq",
retention_period=core.Duration.seconds(1209600),
)
process_new_scenes_queue_alarm = cloudwatch.Alarm(
self,
"ProcessNewScenesQueueAlarm",
metric=self.queues_["process_new_scenes_queue_dlq"].metric(
"ApproximateNumberOfMessagesVisible"),
evaluation_periods=1,
threshold=0.0,
comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD,
)
process_new_scenes_queue_alarm.add_alarm_action(
cw_actions.SnsAction(self.topics_["alarm_topic"]))
self.create_queue(
id="new_scenes_queue",
visibility_timeout=core.Duration.seconds(385),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=1,
queue=self.queues_["process_new_scenes_queue_dlq"]),
)
# Add subscriptions for each CB4 camera
sns.Topic.from_topic_arn(
self,
id="CB4MUX",
topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4MUXQuicklook",
).add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["new_scenes_queue"]))
sns.Topic.from_topic_arn(
self,
id="CB4AWFI",
topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4AWFIQuicklook",
).add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["new_scenes_queue"]))
sns.Topic.from_topic_arn(
self,
id="CB4PAN10M",
topic_arn=
"arn:aws:sns:us-east-1:599544552497:NewCB4PAN10MQuicklook",
).add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["new_scenes_queue"]))
sns.Topic.from_topic_arn(
self,
id="CBPAN5M",
topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4PAN5MQuicklook",
).add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["new_scenes_queue"]))
# Subscription for CB4A (all cameras)
sns.Topic.from_topic_arn(
self,
id="CB4A",
topic_arn="arn:aws:sns:us-east-1:599544552497:NewCB4AQuicklook",
).add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["new_scenes_queue"]))
self.create_queue(
id="catalog_prefix_update_queue",
visibility_timeout=core.Duration.seconds(60),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3, queue=self.queues_["dead_letter_queue"]),
)
# Reconcile queue for INPE's XML metadata
self.create_queue(
id="consume_reconcile_queue_dlq",
retention_period=core.Duration.seconds(1209600),
)
consume_reconcile_queue_alarm = cloudwatch.Alarm(
self,
"ConsumeReconcileQueueAlarm",
metric=self.queues_["consume_reconcile_queue_dlq"].metric(
"ApproximateNumberOfMessagesVisible"),
evaluation_periods=1,
threshold=0.0,
comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD,
)
consume_reconcile_queue_alarm.add_alarm_action(
cw_actions.SnsAction(self.topics_["alarm_topic"]))
self.create_queue(
id="reconcile_queue",
visibility_timeout=core.Duration.seconds(1000),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3,
queue=self.queues_["consume_reconcile_queue_dlq"]),
)
# Reconcile queue for STAC items
self.create_queue(
id="consume_stac_reconcile_queue_dlq",
retention_period=core.Duration.seconds(1209600),
)
consume_stac_reconcile_queue_alarm = cloudwatch.Alarm(
self,
"ConsumeStacReconcileQueueAlarm",
metric=self.queues_["consume_stac_reconcile_queue_dlq"].metric(
"ApproximateNumberOfMessagesVisible"),
evaluation_periods=1,
threshold=0.0,
comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD,
)
consume_stac_reconcile_queue_alarm.add_alarm_action(
cw_actions.SnsAction(self.topics_["alarm_topic"]))
self.create_queue(
id="stac_reconcile_queue",
visibility_timeout=core.Duration.seconds(1000),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3,
queue=self.queues_["consume_stac_reconcile_queue_dlq"],
),
)
# Queue for STAC items to be inserted into Elasticsearch. Subscribe to the
# topic with new stac items
self.create_queue(
id="insert_into_elasticsearch_queue",
visibility_timeout=core.Duration.seconds(180),
retention_period=core.Duration.seconds(1209600),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3, queue=self.queues_["dead_letter_queue"]),
)
# Subscription for new item topics
self.topics_["stac_item_topic"].add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["insert_into_elasticsearch_queue"]))
# Subscription for reconciled item topics
self.topics_["reconcile_stac_item_topic"].add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["insert_into_elasticsearch_queue"]))
# Backup queue for STAC items inserted into Elasticsearch.
# This holds the same items received by "insert_into_elasticsearch_queue",
# simply holding them for some time to allow recover from ES
# cluster failures (see #78)
# This queue subscribe only to new item topics
self.create_queue(
id="backup_insert_into_elasticsearch_queue",
visibility_timeout=core.Duration.seconds(180),
retention_period=core.Duration.days(
settings.backup_queue_retention_days),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3, queue=self.queues_["dead_letter_queue"]),
)
# Subscription for new item topics
self.topics_["stac_item_topic"].add_subscription(
sns_subscriptions.SqsSubscription(
self.queues_["backup_insert_into_elasticsearch_queue"]))
