def __init__(self, scope: core.Construct, id: str, *, prefix: str,
environment: str, configuration, **kwargs):
"""
:param scope: Stack class, used by CDK.
:param id: ID of the construct, used by CDK.
:param prefix: Prefix of the construct, used for naming purposes.
:param environment: Environment of the construct, used for naming purposes.
:param configuration: Configuration of the construct. In this case SQS_CONFIG_SCHEMA.
:param kwargs: Other parameters that could be used by the construct.
"""
super().__init__(scope, id, **kwargs)
self.prefix = prefix
self.environment_ = environment
self._configuration = configuration
# Validating that the payload passed is correct
validate_configuration(configuration_schema=SQS_CONFIG_SCHEMA,
configuration_received=self._configuration)
# Defining SQS Queue
queue_data = deepcopy(self._configuration["queue"])
self._sqs_queue = base_queue(construct=self, **queue_data)
# Validating Lambda Function Runtime
functions_data = self._configuration["lambda_handlers"]
self._lambda_functions = list()
for lambda_function in functions_data:
_lambda_function = base_lambda_function(self, **lambda_function)
self._lambda_functions.append(_lambda_function)
_lambda_function.add_event_source(
lambda_sources.SqsEventSource(queue=self._sqs_queue,
batch_size=10))
def catalog(self, bucket, notification_queue):
catalog_table = _ddb.Table(
self,
"CatalogTable",
billing_mode=_ddb.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY,
partition_key=_ddb.Attribute(name='Source',
type=_ddb.AttributeType.STRING),
sort_key=_ddb.Attribute(name='Timestamp',
type=_ddb.AttributeType.STRING))
event_recorder = _lambda.Function(
self,
"EventRecorder",
handler='lambda_function.lambda_handler',
# https://github.com/aws/aws-cdk/issues/5491
# pylint: disable=no-value-for-parameter
code=_lambda.Code.asset('src/event_recorder'),
runtime=_lambda.Runtime.PYTHON_3_7,
log_retention=_logs.RetentionDays.ONE_MONTH,
environment={
'BUCKET_NAME': bucket.bucket_name,
'TABLE_NAME': catalog_table.table_name,
'TOPIC_SSM_PREFIX':
"/{}/DistributionTopics/".format(self.stack_id)
})
bucket.grant_read(event_recorder)
catalog_table.grant_write_data(event_recorder)
event_recorder.add_event_source(
_lambda_event_sources.SqsEventSource(notification_queue))
return catalog_table, event_recorder
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
queue = _sqs.Queue(self,
"VsamToDynamoQueue",
visibility_timeout=Duration.seconds(300),
queue_name='VsamToDynamoQueue')
dynamoTable = _dyn.Table(
self,
"CLIENT",
partition_key=_dyn.Attribute(name="CLIENT-KEY",
type=_dyn.AttributeType.STRING),
table_name="CLIENT",
)
# Create the Lambda function to subscribe to SQS and store the record in DynamoDB
# The source code is in './src' directory
lambda_fn = _lambda.Function(
self,
"SQSToDynamoFunction",
runtime=_lambda.Runtime.PYTHON_3_9,
handler="insertRecord.handler",
code=_lambda.Code.from_asset("lambda_fns"),
)
dynamoTable.grant_write_data(lambda_fn)
queue.grant_consume_messages(lambda_fn)
lambda_fn.add_event_source(_event.SqsEventSource(queue))
def __init__(self, scope: core.Construct, id: str, sns_topic_arn: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Queue Setup
sqs_queue = sqs.Queue(self, 'RDSPublishQueue', visibility_timeout=core.Duration.seconds(300))
sqs_lambda = _lambda.Function(self, "sqsLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="sqs.handler",
code=_lambda.Code.from_asset("lambda_fns"),
tracing=_lambda.Tracing.ACTIVE,
environment={
"SQS_URL": sqs_queue.queue_url
}
)
sqs_queue.grant_send_messages(sqs_lambda)
topic = sns.Topic.from_topic_arn(self, 'SNSTopic', sns_topic_arn)
topic.add_subscription(subscriptions.LambdaSubscription(sqs_lambda))
sqs_subscribe_lambda = _lambda.Function(self, "sqsSubscribeLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="sqs_subscribe.handler",
code=_lambda.Code.from_asset("lambda_fns"),
tracing=_lambda.Tracing.ACTIVE
)
sqs_queue.grant_consume_messages(sqs_subscribe_lambda)
sqs_subscribe_lambda.add_event_source(lambda_event.SqsEventSource(sqs_queue))
def setLambdaTriggers(self):
## **************** Event Sources & Triggers ****************
# Read Comprehend completion notification and prepare triples for neptune
self.sqs_queue = sqs.Queue.from_queue_arn(
self, "CDKQueue",
self.comprehend_complete_sqs.get_att("Arn").to_string() #f'arn:aws:sqs:us-east-1:{core.Aws.ACCOUNT_ID}:{self.comprehend_complete_sqs.queue_name}'
)
self.generate_triples_lambda.add_event_source(aws_lambda_event_sources.SqsEventSource(self.sqs_queue, batch_size = 1))
def add_reviewer(self):
reviewer = AnalyzerLambda(self,'review-queued-repo',
project_name='review-queued-repo',
concurrency= 50,
datalake=self.datalake)
queue = sqs.Queue(self,'PendingReviewQueue',
visibility_timeout= core.Duration.minutes(15))
reviewer.function.add_event_source(events.SqsEventSource(queue=queue, batch_size=1))
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
queue = _sqs.Queue(self,
"MyQueue",
visibility_timeout=Duration.seconds(300))
# Create the AWS Lambda function to subscribe to Amazon SQS queue
# The source code is in './lambda' directory
lambda_function = _lambda.Function(
self,
"MyLambdaFunction",
runtime=_lambda.Runtime.PYTHON_3_9,
handler="get_messages.handler",
code=_lambda.Code.from_asset("lambda"),
)
# Set Lambda Logs Retention and Removal Policy
logs.LogGroup(
self,
'logs',
log_group_name=f"/aws/lambda/{lambda_function.function_name}",
removal_policy=RemovalPolicy.DESTROY,
retention=logs.RetentionDays.ONE_DAY)
#Grant permission to AWS Lambda function to consume messages from the Amazon SQS queue
queue.grant_consume_messages(lambda_function)
#Configure the Amazon SQS queue to trigger the AWS Lambda function
lambda_function.add_event_source(_event.SqsEventSource(queue))
CfnOutput(self,
"FunctionName",
value=lambda_function.function_name,
export_name='FunctionName',
description='Function name')
CfnOutput(self,
"QueueName",
value=queue.queue_name,
export_name='QueueName',
description='SQS queue name')
CfnOutput(self,
"QueueArn",
value=queue.queue_arn,
export_name='QueueArn',
description='SQS queue ARN')
CfnOutput(self,
"QueueUrl",
value=queue.queue_url,
export_name='QueueUrl',
description='SQS queue URL')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
queue = _sqs.Queue(self, 'HelloQueue')
my_lambda = _lambda.Function(self,
'HelloFunction',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='hello.handler')
my_lambda.add_event_source(event_sources.SqsEventSource(queue))
def create_events(self, services):
# kickoff_notification = aws_s3_notifications.LambdaDestination(services["lambda"]["kickoff"])
extensions = [
"pdf", "pDf", "pDF", "pdF", "PDF", "Pdf",
"png", "pNg", "pNG", "pnG", "PNG", "Png",
"jpg", "jPg", "jPG", "jpG", "JPG", "Jpg"
]
for extension in extensions:
services["main_s3_bucket"].add_event_notification(
aws_s3.EventType.OBJECT_CREATED,
aws_s3_notifications.SqsDestination(services["sf_sqs"]),
aws_s3.NotificationKeyFilter(prefix="uploads/", suffix=extension)
)
services["lambda"]["kickoff"].add_event_source(
aws_lambda_event_sources.SqsEventSource(
services["sf_sqs"],
batch_size=1
)
)
services["lambda"]["analyzepdf"].add_event_source(
aws_lambda_event_sources.SqsEventSource(
services["textract_sqs"],
batch_size=1
)
)
human_complete_target = aws_events_targets.LambdaFunction(services["lambda"]["humancomplete"])
human_review_event_pattern = aws_events.EventPattern(
source=["aws.sagemaker"],
detail_type=["SageMaker A2I HumanLoop Status Change"]
)
aws_events.Rule(self,
"multipadepdfa2i_HumanReviewComplete",
event_pattern=human_review_event_pattern,
targets=[human_complete_target]
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# DynamoDB Table
# This is standing in for what is RDS on the diagram due to simpler/cheaper setup
table = dynamo_db.Table(self,
"Messages",
partition_key=dynamo_db.Attribute(
name="id",
type=dynamo_db.AttributeType.STRING))
# Queue Setup
sqs_queue = sqs.Queue(self,
'RDSPublishQueue',
visibility_timeout=core.Duration.seconds(300))
# defines an AWS Lambda resource to publish to our sqs_queue
sqs_publish_lambda = _lambda.Function(
self,
"SQSPublishLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X, # execution environment
handler="lambda.handler", # file is "lambda", function is "handler"
code=_lambda.Code.from_asset(
"lambda_fns/publish"
), # Code loaded from the lambda_fns/publish dir
environment={'queueURL': sqs_queue.queue_url})
sqs_queue.grant_send_messages(sqs_publish_lambda)
# defines an AWS Lambda resource to pull from our sqs_queue
sqs_subscribe_lambda = _lambda.Function(
self,
"SQSSubscribeLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X, # execution environment
handler="lambda.handler", # file is "lambda", function is "handler"
code=_lambda.Code.from_asset(
"lambda_fns/subscribe"
), # Code loaded from the lambda_fns/subscribe dir
environment={
'queueURL': sqs_queue.queue_url,
'tableName': table.table_name
},
reserved_concurrent_executions=2)
sqs_queue.grant_consume_messages(sqs_subscribe_lambda)
sqs_subscribe_lambda.add_event_source(
lambda_event.SqsEventSource(sqs_queue))
table.grant_read_write_data(sqs_subscribe_lambda)
# defines an API Gateway REST API resource backed by our "sqs_publish_lambda" function.
api_gw.LambdaRestApi(self, 'Endpoint', handler=sqs_publish_lambda)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create Primary S3 Bucket
main_bucket = aws_s3.Bucket(self, "multipagepdfa2i", removal_policy=core.RemovalPolicy.DESTROY)
# Create sqs queue
page_sqs = aws_sqs.Queue(
self, "multipagepdfa2i_page_sqs",
queue_name = "multipagepdfa2i_page_sqs",
visibility_timeout=core.Duration.minutes(3)
)
# Create all of the Lambda Functions
lambda_functions = self.create_lambda_functions(page_sqs)
# Create notification that triggers kick off lambda on pdf being uploaded to kickoff
kickoff_notification = aws_s3_notifications.LambdaDestination(lambda_functions["kickoff"])
lambda_functions["analyzepdf"].add_event_source(aws_lambda_event_sources.SqsEventSource(page_sqs, batch_size=3))
main_bucket.add_event_notification(
aws_s3.EventType.OBJECT_CREATED,
kickoff_notification,
aws_s3.NotificationKeyFilter(prefix="uploads/", suffix="pdf")
)
self.configure_dynamo_table("multia2ipdf_callback", "jobid", "callback_token")
self.configure_dynamo_table("multipagepdfa2i_upload_ids", "id", "key")
self.create_state_machine(lambda_functions, page_sqs)
human_complete_target = aws_events_targets.LambdaFunction(lambda_functions["humancomplete"])
human_review_event_pattern = aws_events.EventPattern(
source=["aws.sagemaker"],
detail_type=["SageMaker A2I HumanLoop Status Change"]
)
aws_events.Rule(self,
"multipadepdfa2i_HumanReviewComplete",
event_pattern=human_review_event_pattern,
targets=[human_complete_target]
)
def __init__(self, scope: core.Construct, id: str, publisher_lambda,
suscriber_lambda, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
queue_fail = sqs.Queue(self,
"queue_fail",
visibility_timeout=core.Duration.seconds(20))
dlq = sqs.DeadLetterQueue(max_receive_count=100, queue=queue_fail)
self.queue = sqs.Queue(self,
"base_queue",
visibility_timeout=core.Duration.seconds(20),
dead_letter_queue=dlq)
#Configuramos la lambda para que reciba los mensajes de la cola
self.queue.grant_consume_messages(suscriber_lambda)
event_source = aws_lambda_event_sources.SqsEventSource(self.queue,
batch_size=1)
suscriber_lambda.add_event_source(event_source)
#la lambda que publica mensajes
self.queue.grant_send_messages(publisher_lambda)
def __init__(self, scope: core.Construct, id: str, *, prefix: str, environment: str, configuration, **kwargs):
"""
:param scope: Stack class, used by CDK.
:param id: ID of the construct, used by CDK.
:param prefix: Prefix of the construct, used for naming purposes.
:param environment: Environment of the construct, used for naming purposes.
:param configuration: Configuration of the construct. In this case IOT_SNS_CONFIG_SCHEMA.
:param kwargs: Other parameters that could be used by the construct.
"""
super().__init__(scope, id, **kwargs)
self.prefix = prefix
self.environment_ = environment
self._configuration = configuration
# Validating that the payload passed is correct
validate_configuration(configuration_schema=IOT_SQS_CONFIG_SCHEMA, configuration_received=self._configuration)
# Defining SQS Queue
queue_data = deepcopy(self._configuration["queue"])
self._sqs_queue = base_queue(construct=self, **queue_data)
# Defining IAM Role
role = base_sqs_role(self, resource_name=queue_data["queue_name"], principal_resource="iot")
# Validating Lambda Function Runtime
functions_data = self._configuration["lambda_handlers"]
self._lambda_functions = list()
for lambda_function in functions_data:
_lambda_function = base_lambda_function(self, **lambda_function)
self._lambda_functions.append(_lambda_function)
_lambda_function.add_event_source(lambda_sources.SqsEventSource(queue=self._sqs_queue, batch_size=10))
# Defining Topic Rule properties
action = iot.CfnTopicRule.SqsActionProperty(queue_url=self._sqs_queue.queue_url, role_arn=role.role_arn)
action_property = iot.CfnTopicRule.ActionProperty(sqs=action)
rule_data = self._configuration["iot_rule"]
self._iot_rule = base_iot_rule(self, action_property=action_property, **rule_data)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Creates Queue
requestqueue = sqs.Queue(
self,
"RequestsQueue",
visibility_timeout=core.Duration.seconds(300),
)
# Create DDB Table
requests_table = ddb.Table(self,
"requests_table",
partition_key=ddb.Attribute(
name="id",
type=ddb.AttributeType.STRING))
# Defines an AWS Lambda resource
hello_lambda = _lambda.Function(
self,
'HelloHandler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='hello.handler',
)
# Adds DN Table as env
hello_lambda.add_environment("TABLE_NAME", requests_table.table_name)
# grant permission to lambda to write to demo table
requests_table.grant_write_data(hello_lambda)
# SQS Event Source for Lambda
sqs_event_source = lambda_event_source.SqsEventSource(requestqueue)
# SQS event source to Function
hello_lambda.add_event_source(sqs_event_source)
def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# Policies
s3_access_policy = iam.ManagedPolicy.from_managed_policy_arn(
self,
id="s3_access_policy",
managed_policy_arn="arn:aws:iam::aws:policy/AmazonS3FullAccess")
lambda_access_policy = iam.ManagedPolicy.from_managed_policy_arn(
self,
id="lambda_access_policy",
managed_policy_arn="arn:aws:iam::aws:policy/AWSLambda_FullAccess")
logs_policy = iam.ManagedPolicy.from_managed_policy_arn(
self,
id="logs_policy",
managed_policy_arn=
"arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")
# Roles
lambda_role = iam.Role(
self,
id="lambda_role",
assumed_by=iam.ServicePrincipal(service="lambda.amazonaws.com"),
managed_policies=[
s3_access_policy, lambda_access_policy, logs_policy
],
role_name=f"midi-to-mp3-lambda-role")
# SQS
conversion_sqs = sqs.Queue(self,
id=f"conversion_sqs",
queue_name=f"conversion_sqs",
visibility_timeout=core.Duration.hours(12),
retention_period=core.Duration.days(1))
kickoff_sqs = sqs.Queue(self,
id=f"kickoff_sqs",
queue_name=f"kickoff_sqs",
visibility_timeout=core.Duration.hours(12),
retention_period=core.Duration.days(1))
# S3
midi_file_dropoff_bucket = s3.Bucket(
self,
id="midi_files_dropoff",
bucket_name=MIDI_FILE_DROPOFF_BUCKET,
auto_delete_objects=True,
removal_policy=core.RemovalPolicy.DESTROY)
midi_file_dropoff_bucket.add_event_notification(
event=s3.EventType.OBJECT_CREATED,
dest=s3n.SqsDestination(kickoff_sqs))
created_mp3_files_bucket = s3.Bucket(
self,
id="created_mp3_files",
bucket_name=CREATED_MP3_FILES_BUCKET,
auto_delete_objects=True,
removal_policy=core.RemovalPolicy.DESTROY)
component_midi_files_bucket = s3.Bucket(
self,
id="component_midi_files",
bucket_name=COMPONENT_MIDI_FILES_BUCKET,
auto_delete_objects=True,
removal_policy=core.RemovalPolicy.DESTROY)
component_midi_files_bucket.add_event_notification(
event=s3.EventType.OBJECT_CREATED,
dest=s3n.SqsDestination(conversion_sqs))
# Lambdas
lambda_code = lambda_.DockerImageCode.from_image_asset(
directory='./midi_to_mp3_lambda/',
file="Dockerfile",
build_args={
"AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID"),
"AWS_SECRET_ACCESS_KEY":
os.environ.get("AWS_SECRET_ACCESS_KEY")
})
midi_to_mp3_lambda = lambda_.DockerImageFunction(
self,
id="midi_to_mp3_lambda",
role=lambda_role,
function_name="midi-to-mp3",
memory_size=1024,
timeout=core.Duration.minutes(5),
code=lambda_code)
midi_split_lambda = alg.GoFunction(
self,
id="midi_split_lambda",
entry="./midi_split_lambda/midi_split_lambda.go",
timeout=core.Duration.minutes(15),
runtime=lambda_.Runtime.GO_1_X,
role=lambda_role,
function_name=f"midi-split-lambda",
memory_size=512,
bundling={"environment": {
"GO111MODULE": "off"
}})
s3_cleanup_lambda = lambda_.Function(
self,
id="s3_cleanup_lambda",
runtime=lambda_.Runtime.PYTHON_3_8,
role=lambda_role,
function_name="s3-cleanup-lambda",
memory_size=256,
timeout=core.Duration.minutes(5),
environment={"NUM_WEEKS_TO_KEEP_FILES": "1"},
handler="s3_cleanup_lambda.handler",
code=lambda_.Code.from_asset(os.path.join(".",
"s3_cleanup_lambda")))
weekly_on_sunday_cron = events.Rule(
self,
"Rule",
schedule=events.Schedule.cron(minute='0',
hour='0',
week_day="SUN",
month='*',
year='*'),
)
weekly_on_sunday_cron.add_target(
targets.LambdaFunction(s3_cleanup_lambda))
# Event Sources
midi_to_mp3_lambda.add_event_source(
eventsources.SqsEventSource(queue=conversion_sqs))
midi_split_lambda.add_event_source(
eventsources.SqsEventSource(queue=kickoff_sqs))
def __init__(self, scope: Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
#Create the SQS queue
queue = sqs.Queue(self, "SQSQueue")
#Create the API GW service role with permissions to call SQS
rest_api_role = iam.Role(
self,
"RestAPIRole",
assumed_by=iam.ServicePrincipal("apigateway.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSQSFullAccess")
])
#Create an API GW Rest API
base_api = apigw.RestApi(self, 'ApiGW', rest_api_name='TestAPI')
base_api.root.add_method("ANY")
#Create a resource named "example" on the base API
api_resource = base_api.root.add_resource('example')
#Create API Integration Response object: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html
integration_response = apigw.IntegrationResponse(
status_code="200",
response_templates={"application/json": ""},
)
#Create API Integration Options object: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html
api_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=[integration_response],
request_templates={
"application/json":
"Action=SendMessage&MessageBody=$input.body"
},
passthrough_behavior=apigw.PassthroughBehavior.NEVER,
request_parameters={
"integration.request.header.Content-Type":
"'application/x-www-form-urlencoded'"
},
)
#Create AWS Integration Object for SQS: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/AwsIntegration.html
api_resource_sqs_integration = apigw.AwsIntegration(
service="sqs",
integration_http_method="POST",
path="{}/{}".format(Aws.ACCOUNT_ID, queue.queue_name),
options=api_integration_options)
#Create a Method Response Object: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/MethodResponse.html
method_response = apigw.MethodResponse(status_code="200")
#Add the API GW Integration to the "example" API GW Resource
api_resource.add_method("POST",
api_resource_sqs_integration,
method_responses=[method_response])
#Creating Lambda function that will be triggered by the SQS Queue
sqs_lambda = _lambda.Function(
self,
'SQSTriggerLambda',
handler='lambda-handler.handler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset('lambda'),
)
#Create an SQS event source for Lambda
sqs_event_source = lambda_event_source.SqsEventSource(queue)
#Add SQS event source to the Lambda function
sqs_lambda.add_event_source(sqs_event_source)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
env_name = self.node.try_get_context('env')
#Create the SQS queue
queue = sqs.Queue(self,
id=f"{env_name}-SQSQueue",
queue_name=f"{env_name}-queue")
#Create the API GW service role with permissions to call SQS
rest_api_role = iam.Role(
self,
id=f"{env_name}-RestAPISQSRole",
assumed_by=iam.ServicePrincipal("apigateway.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSQSFullAccess")
])
#Create an API GW Rest API
base_api = apigw.RestApi(
self,
id=f'{env_name}-ApiGW',
rest_api_name=f'{env_name}SQSTestAPI',
api_key_source_type=apigw.ApiKeySourceType.HEADER)
usage_api_key_value = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(40))
usage_api_key = base_api.add_api_key(id=f'{env_name}-apikey',
value=usage_api_key_value)
usage_plan = base_api.add_usage_plan(id=f'{env_name}-usageplan',
name=f'{env_name}-usageplan',
api_key=usage_api_key,
throttle=apigw.ThrottleSettings(
rate_limit=10, burst_limit=2))
usage_plan.add_api_stage(stage=base_api.deployment_stage)
#Create a resource named "example" on the base API
api_resource = base_api.root.add_resource('sqstest')
#Create API Integration Response object: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html
integration_response = apigw.IntegrationResponse(
status_code="200",
response_templates={"application/json": ""},
)
#Create API Integration Options object: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html
api_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=[integration_response],
request_templates={
"application/json":
"Action=SendMessage&MessageBody=$input.body"
},
passthrough_behavior=apigw.PassthroughBehavior.NEVER,
request_parameters={
"integration.request.header.Content-Type":
"'application/x-www-form-urlencoded'"
},
)
#Create AWS Integration Object for SQS: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/AwsIntegration.html
api_resource_sqs_integration = apigw.AwsIntegration(
service="sqs",
integration_http_method="POST",
# must be ACCOUNT_ID. Just the way URL to SQS is created
path="{}/{}".format(core.Aws.ACCOUNT_ID, queue.queue_name),
options=api_integration_options)
#Create a Method Response Object: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/MethodResponse.html
method_response = apigw.MethodResponse(status_code="200")
#Add the API GW Integration to the "example" API GW Resource
api_resource.add_method("POST",
api_resource_sqs_integration,
method_responses=[method_response],
api_key_required=True)
#Creating Lambda function that will be triggered by the SQS Queue
sqs_lambda = _lambda.Function(
self,
'SQSTriggerLambda',
handler='sqs_lambda.handler',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.asset('pr_sqs_lambda'),
)
#Create an SQS event source for Lambda
sqs_event_source = lambda_event_source.SqsEventSource(queue)
#Add SQS event source to the Lambda function
sqs_lambda.add_event_source(sqs_event_source)
# https://67ixnggm81.execute-api.us-east-1.amazonaws.com/prod/sqstest
region = core.Aws.REGION
core.CfnOutput(self,
'api-gw-url',
value='https://' + base_api.rest_api_id +
'.execute-api.' + region +
'.amazonaws.com/prod/sqstest',
export_name='api-sqs-gw-url')
print(f'API Key: {usage_api_key_value}')
"""
def __init__(
self,
scope: core.Construct,
id: str,
create_dependency_layer: Callable[[], None],
**kwargs
) -> None:
super().__init__(scope, id, **kwargs)
# create dependency layer zip for lambda function
create_dependency_layer()
api_secret = secretsmanager.Secret(
self,
"ActualApiSecret",
description="Secrets required to communicate with Aarogya Setu OpenAPI",
)
# create cognito user pool for authentication
user_pool = cognito.UserPool(
self,
"AppUserPool",
self_sign_up_enabled=True,
account_recovery=cognito.AccountRecovery.PHONE_AND_EMAIL,
user_verification=cognito.VerificationEmailStyle.CODE,
auto_verify={"email": True},
standard_attributes={"email": {"required": True, "mutable": True}},
)
user_pool_client = cognito.UserPoolClient(
self, "UserPoolClient", user_pool=user_pool
)
# Create storage and queue
bulk_request_queue = sqs.Queue(
self,
"BulkRequestQueue",
)
user_status_table = ddb.Table(
self,
"UserStatusTable",
partition_key={"name": "mobile_number", "type": ddb.AttributeType.STRING},
time_to_live_attribute="expdate",
)
self._user_status_table = user_status_table
requests_table = ddb.Table(
self,
"RequestsTable",
partition_key={"name": "mobile_number", "type": ddb.AttributeType.STRING},
time_to_live_attribute="expdate",
)
# Create layer for lambda run time dependencies
dependency_layer = _lambda.LayerVersion(
self,
"PythonDependencies",
code=_lambda.Code.from_asset(path.join("lambda", "dependency-layer.zip")),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
description="The layer contains requests and pyjwt dependencies",
)
# Create Lambda functions
single_request = _lambda.Function(
self,
"SingleRequesetHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="single_request.handler",
timeout=core.Duration.seconds(10),
layers=[dependency_layer],
environment={
"USER_STATUS_TABLE": user_status_table.table_name,
"REQUESTS_TABLE": requests_table.table_name,
"API_SECRET_ARN": api_secret.secret_full_arn,
},
)
# give lambda access permissions to ddb tables and secrets
user_status_table.grant_read_write_data(single_request)
requests_table.grant_read_write_data(single_request)
api_secret.grant_read(single_request)
bulk_request = _lambda.Function(
self,
"BulkRequestHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="bulk_request.handler",
timeout=core.Duration.seconds(30),
environment={
"QUEUE_URL": bulk_request_queue.queue_url,
},
)
# give lambda access to write to queue
bulk_request_queue.grant_send_messages(bulk_request)
queue_receiver = _lambda.Function(
self,
"QueueReceiverHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="queue_receiver.handler",
timeout=core.Duration.seconds(10),
layers=[dependency_layer],
environment={
"USER_STATUS_TABLE": user_status_table.table_name,
"REQUESTS_TABLE": requests_table.table_name,
"QUEUE_URL": bulk_request_queue.queue_url,
"API_SECRET_ARN": api_secret.secret_full_arn,
},
)
# lambda gets triggered by sqs queue and writes to both tables
queue_receiver.add_event_source(
events.SqsEventSource(bulk_request_queue, batch_size=1)
)
# give queue receiver access to tables, queue and secrets
bulk_request_queue.grant_consume_messages(queue_receiver)
user_status_table.grant_read_write_data(queue_receiver)
requests_table.grant_read_write_data(queue_receiver)
api_secret.grant_read(queue_receiver)
scan_table = _lambda.Function(
self,
"ScanTableHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="scan_table.handler",
timeout=core.Duration.seconds(30),
environment={
"USER_STATUS_TABLE": user_status_table.table_name,
},
)
user_status_table.grant_read_data(scan_table)
# create api endpoints with authorization
api = apigw.RestApi(
self,
"ASetuApiGateway",
default_cors_preflight_options=apigw.CorsOptions(
allow_origins=apigw.Cors.ALL_ORIGINS
),
)
auth = apigw.CfnAuthorizer(
self,
"ApiCognitoAuthorizer",
name="CognitoAuthorizer",
type="COGNITO_USER_POOLS",
authorizer_result_ttl_in_seconds=300,
identity_source="method.request.header.Authorization",
rest_api_id=api.rest_api_id,
provider_arns=[user_pool.user_pool_arn],
)
single_request_integration = apigw.LambdaIntegration(single_request, proxy=True)
single_request_resource = api.root.add_resource("status")
single_method = single_request_resource.add_method(
"POST",
single_request_integration,
api_key_required=False,
authorizer=auth,
authorization_type=apigw.AuthorizationType.COGNITO,
)
bulk_request_integration = apigw.LambdaIntegration(bulk_request, proxy=True)
bulk_request_resource = api.root.add_resource("bulk_status")
bulk_method = bulk_request_resource.add_method(
"POST",
bulk_request_integration,
api_key_required=False,
authorizer=auth,
authorization_type=apigw.AuthorizationType.COGNITO,
)
scan_table_integration = apigw.LambdaIntegration(scan_table, proxy=True)
scan_table_resource = api.root.add_resource("scan")
scan_method = scan_table_resource.add_method(
"GET",
scan_table_integration,
api_key_required=False,
authorizer=auth,
authorization_type=apigw.AuthorizationType.COGNITO,
)
# Override authorizer to use COGNITO to authorize apis
# Solution from: https://github.com/aws/aws-cdk/issues/9023#issuecomment-658309644
methods = [single_method, bulk_method, scan_method]
for method in methods:
method.node.find_child("Resource").add_property_override(
"AuthorizationType", "COGNITO_USER_POOLS"
)
method.node.find_child("Resource").add_property_override(
"AuthorizerId", {"Ref": auth.logical_id}
)
# Export output values for frontend application
core.CfnOutput(
self,
"user-pool-id",
value=user_pool.user_pool_id,
export_name="USER-POOL-ID",
)
core.CfnOutput(
self,
"user-pool-web-client",
value=user_pool_client.user_pool_client_id,
export_name="WEB-CLIENT-ID",
)
core.CfnOutput(
self, "api-endpoint-url", value=api.url, export_name="API-ENDPOINT-URL"
)
core.CfnOutput(
self,
"deployment-region",
value=self.region,
export_name="REGION",
)
core.CfnOutput(
self, "stack-name", value=self.stack_name, export_name="STACK-NAME"
)
core.CfnOutput(
self,
"api-secret-arn",
value=api_secret.secret_full_arn,
export_name="API-SECRET-ARN",
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create image bucket
image_bucket = s3.Bucket(self, 'inbound_image_s3_bucket')
# Create the image processing queue
image_process_queue = sqs.Queue(
self, "image_process_queue",
visibility_timeout=core.Duration.seconds(300),
retention_period=core.Duration.days(1)
)
# Create the image response queue
response_queue = sqs.Queue(
self, "results_queue",
visibility_timeout=core.Duration.seconds(300),
retention_period=core.Duration.days(1)
)
# Set the put object notification to the SQS Queue
image_bucket.add_event_notification(event=s3.EventType.OBJECT_CREATED_PUT,
dest=s3n.SqsDestination(image_process_queue))
# Define the AWS Lambda to call Amazon Rekognition DetectFaces
detect_faces_lambda = _lambda.Function(self, 'detect_faces',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='detect_faces.lambda_handler',
code=_lambda.Code.asset('./lambda'),
timeout=core.Duration.seconds(30),
environment={'SQS_RESPONSE_QUEUE': response_queue.queue_name},
reserved_concurrent_executions=50
)
# Set SQS image_process_queue Queue as event source for detect_faces_lambda
detect_faces_lambda.add_event_source(_lambda_events.SqsEventSource(image_process_queue,
batch_size=1))
# Allow response queue messages from lambda
response_queue.grant_send_messages(detect_faces_lambda)
# Allow lambda to call Rekognition by adding a IAM Policy Statement
detect_faces_lambda.add_to_role_policy(iam.PolicyStatement(actions=['rekognition:*'],
resources=['*']))
# Allow lambda to read from S3
image_bucket.grant_read(detect_faces_lambda)
# Define the DynamoDB Table
results_table = dynamodb.Table(self, 'detect_faces_results',
table_name='detect_faces_results',
partition_key=dynamodb.Attribute(name='id', type=dynamodb.AttributeType.STRING),
read_capacity=200,
write_capacity=200
)
# Define the AWS Lambda to write results into DyanamoDB results_table
write_results_lambda = _lambda.Function(self, 'write_results',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='write_results.lambda_handler',
code=_lambda.Code.asset('./lambda'),
timeout=core.Duration.seconds(30),
environment={'TABLE_NAME': results_table.table_name}
)
# Set SQS response_queue Queue as event source for write_results_lambda results_table
write_results_lambda.add_event_source(_lambda_events.SqsEventSource(response_queue,
batch_size=1))
# Allow AWS Lambda write_results_lambda to Write to Dynamodb
results_table.grant_write_data(write_results_lambda)
# Allow AWS Lambda write_results_lambda to read messages from the SQS response_queue Queue
response_queue.grant_consume_messages(write_results_lambda)
# Output to Amazon S3 Image Bucket
core.CfnOutput(self, 'cdk_output',
value=image_bucket.bucket_name,
description='Input Amazon S3 Image Bucket')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.current_dir = os.path.dirname(__file__)
self.bucket = s3.Bucket(
self,
"qs-migration-bucket",
bucket_name=f'quicksight-migration-{core.Aws.ACCOUNT_ID}',
block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
)
self.quicksight_migration_lambda_role = iam.Role(
self,
'quicksight-migration-lambda-role',
description='Role for the Quicksight dashboard migration Lambdas',
role_name='quicksight-migration-lambda-role',
max_session_duration=core.Duration.seconds(3600),
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'AllowAccess':
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'logs:CreateLogGroup', 'logs:CreateLogStream',
'logs:PutLogEvents'
],
resources=[
f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:*'
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["sts:AssumeRole", "iam:ListRoles"],
resources=[
"arn:aws:iam::*:role/quicksight-migration-*-assume-role"
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["s3:PutObject", "s3:ListBucket"],
resources=[
self.bucket.bucket_arn,
f"{self.bucket.bucket_arn}/*"
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["secrets:GetSecretValue"],
resources=[
f"arn:aws:secretsmanager:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:secret:*"
]),
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=[
"quicksight:*",
],
resources=["*"])
])
})
# API Gateway to SQS
self.apigw_sqs = ApiGatewayToSqs(
self,
"ApiGatewayToSQSqsMigration",
allow_create_operation=True,
allow_read_operation=False,
allow_delete_operation=False,
api_gateway_props=apigw.RestApiProps(
rest_api_name="quicksight-migration-sqs",
deploy=True,
default_method_options=apigw.MethodOptions(
authorization_type=apigw.AuthorizationType.NONE),
default_cors_preflight_options=apigw.CorsOptions(
allow_origins=apigw.Cors.ALL_ORIGINS,
allow_methods=apigw.Cors.ALL_METHODS,
allow_headers=[
'Access-Control-Allow-Origin',
'Access-Control-Allow-Headers', 'Content-Type'
]),
policy=iam.PolicyDocument(statements=[
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=['execute-api:Invoke'],
resources=["execute-api:/prod/*"],
principals=[iam.ArnPrincipal("*")])
])),
queue_props=sqs.QueueProps(
queue_name="quicksight-migration-sqs",
visibility_timeout=core.Duration.minutes(15)))
self.quicksight_migration_lambda = _lambda.Function(
self,
'quicksight-migration-lambda',
handler='quicksight_migration.lambda_function.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
os.path.join(self.current_dir,
'../lambda/quicksight_migration/')),
function_name='quicksight_migration_lambda',
role=self.quicksight_migration_lambda_role,
timeout=core.Duration.minutes(15),
memory_size=1024,
environment={
'BUCKET_NAME': self.bucket.bucket_name,
'S3_KEY': 'None',
'INFRA_CONFIG_PARAM': '/infra/config',
'SQS_URL': self.apigw_sqs.sqs_queue.queue_url
})
self.quicksight_migration_lambda.add_event_source(
event_sources.SqsEventSource(
enabled=True,
queue=self.apigw_sqs.sqs_queue,
))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
with open("stack/config.yml", 'r') as stream:
configs = yaml.safe_load(stream)
### S3 core
images_S3_bucket = _s3.Bucket(self, "ICS_IMAGES")
images_S3_bucket.add_cors_rule(
allowed_methods=[_s3.HttpMethods.POST],
allowed_origins=["*"] # add API gateway web resource URL
)
### SQS core
image_deadletter_queue = _sqs.Queue(self, "ICS_IMAGES_DEADLETTER_QUEUE")
image_queue = _sqs.Queue(self, "ICS_IMAGES_QUEUE",
dead_letter_queue={
"max_receive_count": configs["DeadLetterQueue"]["MaxReceiveCount"],
"queue": image_deadletter_queue
})
### api gateway core
api_gateway = RestApi(self, 'ICS_API_GATEWAY', rest_api_name='ImageContentSearchApiGateway')
api_gateway_resource = api_gateway.root.add_resource(configs["ProjectName"])
api_gateway_landing_page_resource = api_gateway_resource.add_resource('web')
api_gateway_get_signedurl_resource = api_gateway_resource.add_resource('signedUrl')
api_gateway_image_search_resource = api_gateway_resource.add_resource('search')
### landing page function
get_landing_page_function = Function(self, "ICS_GET_LANDING_PAGE",
function_name="ICS_GET_LANDING_PAGE",
runtime=Runtime.PYTHON_3_7,
handler="main.handler",
code=Code.asset("./src/landingPage"))
get_landing_page_integration = LambdaIntegration(
get_landing_page_function,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}])
api_gateway_landing_page_resource.add_method('GET', get_landing_page_integration,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}])
### cognito
required_attribute = _cognito.StandardAttribute(required=True)
users_pool = _cognito.UserPool(self, "ICS_USERS_POOL",
auto_verify=_cognito.AutoVerifiedAttrs(email=True), #required for self sign-up
standard_attributes=_cognito.StandardAttributes(email=required_attribute), #required for self sign-up
self_sign_up_enabled=configs["Cognito"]["SelfSignUp"])
user_pool_app_client = _cognito.CfnUserPoolClient(self, "ICS_USERS_POOL_APP_CLIENT",
supported_identity_providers=["COGNITO"],
allowed_o_auth_flows=["implicit"],
allowed_o_auth_scopes=configs["Cognito"]["AllowedOAuthScopes"],
user_pool_id=users_pool.user_pool_id,
callback_ur_ls=[api_gateway_landing_page_resource.url],
allowed_o_auth_flows_user_pool_client=True,
explicit_auth_flows=["ALLOW_REFRESH_TOKEN_AUTH"])
user_pool_domain = _cognito.UserPoolDomain(self, "ICS_USERS_POOL_DOMAIN",
user_pool=users_pool,
cognito_domain=_cognito.CognitoDomainOptions(domain_prefix=configs["Cognito"]["DomainPrefix"]))
### get signed URL function
get_signedurl_function = Function(self, "ICS_GET_SIGNED_URL",
function_name="ICS_GET_SIGNED_URL",
environment={
"ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name,
"DEFAULT_SIGNEDURL_EXPIRY_SECONDS": configs["Functions"]["DefaultSignedUrlExpirySeconds"]
},
runtime=Runtime.PYTHON_3_7,
handler="main.handler",
code=Code.asset("./src/getSignedUrl"))
get_signedurl_integration = LambdaIntegration(
get_signedurl_function,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}])
api_gateway_get_signedurl_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER",
rest_api_id=api_gateway_get_signedurl_resource.rest_api.rest_api_id,
name="ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER",
type="COGNITO_USER_POOLS",
identity_source="method.request.header.Authorization",
provider_arns=[users_pool.user_pool_arn])
api_gateway_get_signedurl_resource.add_method('GET', get_signedurl_integration,
authorization_type=AuthorizationType.COGNITO,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}]
).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_get_signedurl_authorizer.ref)
images_S3_bucket.grant_put(get_signedurl_function, objects_key_pattern="new/*")
### image massage function
image_massage_function = Function(self, "ICS_IMAGE_MASSAGE",
function_name="ICS_IMAGE_MASSAGE",
timeout=core.Duration.seconds(6),
runtime=Runtime.PYTHON_3_7,
environment={"ICS_IMAGE_MASSAGE": image_queue.queue_name},
handler="main.handler",
code=Code.asset("./src/imageMassage"))
images_S3_bucket.grant_write(image_massage_function, "processed/*")
images_S3_bucket.grant_delete(image_massage_function, "new/*")
images_S3_bucket.grant_read(image_massage_function, "new/*")
new_image_added_notification = _s3notification.LambdaDestination(image_massage_function)
images_S3_bucket.add_event_notification(_s3.EventType.OBJECT_CREATED,
new_image_added_notification,
_s3.NotificationKeyFilter(prefix="new/")
)
image_queue.grant_send_messages(image_massage_function)
### image analyzer function
image_analyzer_function = Function(self, "ICS_IMAGE_ANALYSIS",
function_name="ICS_IMAGE_ANALYSIS",
runtime=Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(10),
environment={
"ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name,
"DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"],
"REGION": core.Aws.REGION,
},
handler="main.handler",
code=Code.asset("./src/imageAnalysis"))
image_analyzer_function.add_event_source(_lambda_event_source.SqsEventSource(queue=image_queue, batch_size=10))
image_queue.grant_consume_messages(image_massage_function)
lambda_rekognition_access = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=["rekognition:DetectLabels", "rekognition:DetectModerationLabels"],
resources=["*"]
)
image_analyzer_function.add_to_role_policy(lambda_rekognition_access)
images_S3_bucket.grant_read(image_analyzer_function, "processed/*")
### API gateway finalizing
self.add_cors_options(api_gateway_get_signedurl_resource)
self.add_cors_options(api_gateway_landing_page_resource)
self.add_cors_options(api_gateway_image_search_resource)
### database
database_secret = _secrets_manager.Secret(self, "ICS_DATABASE_SECRET",
secret_name="rds-db-credentials/image-content-search-rds-secret",
generate_secret_string=_secrets_manager.SecretStringGenerator(
generate_string_key='password',
secret_string_template='{"username": "******"}',
exclude_punctuation=True,
exclude_characters='/@\" \\\'',
require_each_included_type=True
)
)
database = _rds.CfnDBCluster(self, "ICS_DATABASE",
engine=_rds.DatabaseClusterEngine.aurora_mysql(version=_rds.AuroraMysqlEngineVersion.VER_5_7_12).engine_type,
engine_mode="serverless",
database_name=configs["Database"]["Name"],
enable_http_endpoint=True,
deletion_protection=configs["Database"]["DeletionProtection"],
master_username=database_secret.secret_value_from_json("username").to_string(),
master_user_password=database_secret.secret_value_from_json("password").to_string(),
scaling_configuration=_rds.CfnDBCluster.ScalingConfigurationProperty(
auto_pause=configs["Database"]["Scaling"]["AutoPause"],
min_capacity=configs["Database"]["Scaling"]["Min"],
max_capacity=configs["Database"]["Scaling"]["Max"],
seconds_until_auto_pause=configs["Database"]["Scaling"]["SecondsToAutoPause"]
),
)
database_cluster_arn = "arn:aws:rds:{}:{}:cluster:{}".format(core.Aws.REGION, core.Aws.ACCOUNT_ID, database.ref)
secret_target = _secrets_manager.CfnSecretTargetAttachment(self,"ICS_DATABASE_SECRET_TARGET",
target_type="AWS::RDS::DBCluster",
target_id=database.ref,
secret_id=database_secret.secret_arn
)
secret_target.node.add_dependency(database)
### database function
image_data_function_role = _iam.Role(self, "ICS_IMAGE_DATA_FUNCTION_ROLE",
role_name="ICS_IMAGE_DATA_FUNCTION_ROLE",
assumed_by=_iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
_iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaVPCAccessExecutionRole"),
_iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"),
_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonRDSDataFullAccess")
]
)
image_data_function = Function(self, "ICS_IMAGE_DATA",
function_name="ICS_IMAGE_DATA",
runtime=Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(5),
role=image_data_function_role,
environment={
"DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"],
"CLUSTER_ARN": database_cluster_arn,
"CREDENTIALS_ARN": database_secret.secret_arn,
"DB_NAME": database.database_name,
"REGION": core.Aws.REGION
},
handler="main.handler",
code=Code.asset("./src/imageData")
)
image_search_integration = LambdaIntegration(
image_data_function,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}])
api_gateway_image_search_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER",
rest_api_id=api_gateway_image_search_resource.rest_api.rest_api_id,
name="ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER",
type="COGNITO_USER_POOLS",
identity_source="method.request.header.Authorization",
provider_arns=[users_pool.user_pool_arn])
api_gateway_image_search_resource.add_method('POST', image_search_integration,
authorization_type=AuthorizationType.COGNITO,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}]
).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_image_search_authorizer.ref)
lambda_access_search = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=["translate:TranslateText"],
resources=["*"]
)
image_data_function.add_to_role_policy(lambda_access_search)
### custom resource
lambda_provider = Provider(self, 'ICS_IMAGE_DATA_PROVIDER',
on_event_handler=image_data_function
)
core.CustomResource(self, 'ICS_IMAGE_DATA_RESOURCE',
service_token=lambda_provider.service_token,
pascal_case_properties=False,
resource_type="Custom::SchemaCreation",
properties={
"source": "Cloudformation"
}
)
### event bridge
event_bus = _events.EventBus(self, "ICS_IMAGE_CONTENT_BUS")
event_rule = _events.Rule(self, "ICS_IMAGE_CONTENT_RULE",
rule_name="ICS_IMAGE_CONTENT_RULE",
description="The event from image analyzer to store the data",
event_bus=event_bus,
event_pattern=_events.EventPattern(resources=[image_analyzer_function.function_arn]),
)
event_rule.add_target(_event_targets.LambdaFunction(image_data_function))
event_bus.grant_put_events(image_analyzer_function)
image_analyzer_function.add_environment("EVENT_BUS", event_bus.event_bus_name)
### outputs
core.CfnOutput(self, 'CognitoHostedUILogin',
value='https://{}.auth.{}.amazoncognito.com/login?client_id={}&response_type=token&scope={}&redirect_uri={}'.format(user_pool_domain.domain_name, core.Aws.REGION, user_pool_app_client.ref, '+'.join(user_pool_app_client.allowed_o_auth_scopes), api_gateway_landing_page_resource.url),
description='The Cognito Hosted UI Login Page'
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
"""Initializes the stack
:param scope: parent of this stack, scope within which resources defined here are accessible
:type file_loc: Optional[aws_cdk.core.Construct]
:param id: the id of the stack
:type id: Optional[str]
:param **kwargs: additional optional arguments
``description``:
a description of the stack (`Optional[str]`).
``env``
AWS environment (account/region) where this stack will be deployed (`Optional[aws_cdk.core.Environment]`).
``stack_name``
name with which to deploy the stack (`Optional[str]`).
``synthesizer``
synthesis method to use while deploying this stack (`Optional[aws_cdk.core.IStackSynthesizer]`).
``tags``
stack tags that will be applied to all taggable resources as well as the stack (`Optional[Mapping[str, str]]`).
``termination_protection``
whether to enable termination protection for this stack (`Optional[bool]`).
"""
super().__init__(scope, id, **kwargs)
metric_handler_dict, webhook_creator_dict = self.handle_parameters()
# timeout used for lambda and sqs, in seconds
lambda_timeout = 300
if self.node.try_get_context('lambda_timeout'):
lambda_timeout = int(self.node.try_get_context('lambda_timeout'))
dead_letter_queue = sqs.Queue(self,
'DeadLetterQueue',
queue_name='DeadLetterQueue')
webhook_queue = sqs.Queue(
self,
'WebhookQueue',
queue_name='WebhookQueue',
visibility_timeout=core.Duration.seconds(lambda_timeout),
dead_letter_queue=sqs.DeadLetterQueue(max_receive_count=3,
queue=dead_letter_queue))
metric_handler_dict['queue_url'] = webhook_queue.queue_url
metric_handler_management_role = self.create_lambda_role_and_policy(
'MetricHandlerManagementRole', [
'cloudwatch:GetDashboard', 'cloudwatch:GetMetricData',
'cloudwatch:ListDashboards', 'cloudwatch:PutDashboard',
'cloudwatch:PutMetricData', 'logs:CreateLogGroup',
'logs:CreateLogStream', 'logs:PutLogEvents',
'secretsmanager:GetSecretValue'
])
metric_handler_timeout = lambda_timeout
metric_handler_function = _lambda.Function(
self,
'MetricsHandler',
function_name='MetricsHandler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda_dir'),
handler='cloudwatch_dashboard_handler.handler',
role=metric_handler_management_role,
environment=metric_handler_dict,
timeout=core.Duration.seconds(metric_handler_timeout))
self.create_event_with_permissions(metric_handler_function)
# Connect SQS to Lambda
sqs_event_source = lambda_event_source.SqsEventSource(webhook_queue)
metric_handler_function.add_event_source(sqs_event_source)
apigw_webhook_url = self.create_and_integrate_apigw(
webhook_queue, metric_handler_dict['dashboard_name_prefix'])
webhook_creator_dict['apigw_endpoint'] = apigw_webhook_url
webhook_role = self.create_lambda_role_and_policy(
'WebhookCreatorRole', ['secretsmanager:GetSecretValue'])
_lambda.Function(self,
'WebhookCreator',
function_name='WebhookCreator',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda_dir'),
handler='webhook_creator.handler',
role=webhook_role,
environment=webhook_creator_dict,
timeout=core.Duration.seconds(5))
def __init__(self, app: core.App, id: str) -> None:
super().__init__(app, id)
##################################
# Lambda Timeouts (seconds) & Queue Redrive
##################################
lambda_gatherer_timeout = 600
lambda_joiner_timeout = 350
# pa11y's timeout is set to 50, so the lambda is just a little longer
lambda_a11y_scan_timeout = 55
max_receive_count = 2
##################################
# S3 Bucket with Domains
##################################
asset = aws_s3_assets.Asset(
self, 'domain-list', path=os.path.abspath('./domains/domains.csv'))
##################################
# Domain Gatherer Lambda and Queue
##################################
domain_queue = sqs.Queue(
self,
'domain-queue',
visibility_timeout=core.Duration.seconds(
(max_receive_count + 1) * lambda_gatherer_timeout),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=max_receive_count,
queue=sqs.Queue(self,
'domain-queue-dlq',
retention_period=core.Duration.days(5))))
lambda_gatherer = lambda_.Function(
self,
"domain-gatherer",
code=lambda_.Code.from_asset('./lambdas/domain_gatherer'),
handler="handler.main",
timeout=core.Duration.seconds(lambda_gatherer_timeout),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=150)
lambda_gatherer.add_environment('SQS_URL', domain_queue.queue_url)
lambda_gatherer.add_environment('BUCKET_NAME', asset.s3_bucket_name)
lambda_gatherer.add_environment('OBJECT_KEY', asset.s3_object_key)
lambda_gatherer_sqs_exec_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'lambda:InvokeFunction', 'sqs:SendMessage',
'sqs:DeleteMessage', 'sqs:SendMessageBatch',
'sqs:SetQueueAttributes', 'sqs:GetQueueAttributes',
'sqs:GetQueueUrl', 'sqs:GetQueueAttributes'
],
resources=[domain_queue.queue_arn])
lambda_gatherer.add_to_role_policy(lambda_gatherer_sqs_exec_policy)
domain_queue.grant_send_messages(lambda_gatherer)
# trigger for 1st and 15th of the month at 18:00 UTC (1pm EST)
lambda_gatherer_rule = events.Rule(self,
"Lambda Gatherer Rule",
schedule=events.Schedule.cron(
minute='0',
hour='18',
day="1,15",
month='*',
year='*'))
lambda_gatherer_rule.add_target(
targets.LambdaFunction(lambda_gatherer))
asset.grant_read(lambda_gatherer)
##################################
# A11y Scanner Lambda and S3
##################################
layer = lambda_.LayerVersion(
self,
'chrome-aws-lambda',
code=lambda_.Code.from_asset('./lambdas/chrome_aws_lambda.zip'),
compatible_runtimes=[lambda_.Runtime.NODEJS_12_X],
description='A layer of chrome-aws-lambda')
lambda_a11y_scan = lambda_.Function(
self,
"a11y-scan",
code=lambda_.Code.from_asset('./lambdas/a11y_scan'),
handler="index.handler",
timeout=core.Duration.seconds(lambda_a11y_scan_timeout),
runtime=lambda_.Runtime.NODEJS_12_X,
memory_size=1000,
layers=[layer])
lambda_a11y_scan.add_event_source(
sources.SqsEventSource(domain_queue, batch_size=1))
# create s3 bucket to put results
results_bucket = s3.Bucket(self,
'results-bucket',
versioned=False,
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True),
lifecycle_rules=[
s3.LifecycleRule(
enabled=True,
expiration=core.Duration.days(10))
])
lambda_a11y_scan.add_environment('BUCKET_NAME',
results_bucket.bucket_name)
results_bucket.grant_put(lambda_a11y_scan)
##################################
# Results Joiner Lambda
##################################
# create s3 bucket to put site data
data_bucket = s3.Bucket(self,
'data-bucket',
versioned=False,
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True))
lambda_joiner = lambda_.Function(
self,
"results-joiner",
code=lambda_.Code.from_asset(
'./lambda-releases/results_joiner.zip'),
handler="handler.main",
timeout=core.Duration.seconds(lambda_joiner_timeout),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=400)
lambda_joiner.add_environment('DATA_BUCKET_NAME',
data_bucket.bucket_name)
lambda_joiner.add_environment('RESULTS_BUCKET_NAME',
results_bucket.bucket_name)
results_bucket.grant_read_write(lambda_joiner)
data_bucket.grant_read_write(lambda_joiner)
# trigger for 8th and 23rd of the month at 18:00 UTC (1pm EST)
lambda_joiner_rule = events.Rule(self,
"Lambda Joiner Rule",
schedule=events.Schedule.cron(
minute='0',
hour='18',
day="8,23",
month='*',
year='*'))
lambda_joiner_rule.add_target(targets.LambdaFunction(lambda_joiner))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
if self.node.try_get_context('vpc_type'):
validate_cdk_json(self)
ES_LOADER_TIMEOUT = 600
######################################################################
# REGION mapping / ELB & Lambda Arch
######################################################################
elb_id_temp = region_info.FactName.ELBV2_ACCOUNT
elb_map_temp = region_info.RegionInfo.region_map(elb_id_temp)
region_dict = {}
for region in elb_map_temp:
# ELB account ID
region_dict[region] = {'ElbV2AccountId': elb_map_temp[region]}
# Lambda Arch
if region in ('us-east-1', 'us-east-2', 'us-west-2', 'ap-south-1',
'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1',
'eu-central-1', 'eu-west-1', 'eu-west-2'):
region_dict[region]['LambdaArch'] = (
aws_lambda.Architecture.ARM_64.name)
else:
region_dict[region]['LambdaArch'] = (
aws_lambda.Architecture.X86_64.name)
region_mapping = core.CfnMapping(
scope=self, id='RegionMap', mapping=region_dict)
######################################################################
# get params
######################################################################
allow_source_address = core.CfnParameter(
self, 'AllowedSourceIpAddresses', allowed_pattern=r'^[0-9./\s]*',
description='Space-delimited list of CIDR blocks',
default='10.0.0.0/8 172.16.0.0/12 192.168.0.0/16')
sns_email = core.CfnParameter(
self, 'SnsEmail', allowed_pattern=r'^[0-9a-zA-Z@_\-\+\.]*',
description=('Input your email as SNS topic, where Amazon '
'OpenSearch Service will send alerts to'),
default='*****@*****.**')
geoip_license_key = core.CfnParameter(
self, 'GeoLite2LicenseKey', allowed_pattern=r'^[0-9a-zA-Z]{16}$',
default='xxxxxxxxxxxxxxxx',
description=("If you wolud like to enrich geoip locaiton such as "
"IP address's country, get a license key form MaxMind"
" and input the key. If you not, keep "
"xxxxxxxxxxxxxxxx"))
reserved_concurrency = core.CfnParameter(
self, 'ReservedConcurrency', default=10, type='Number',
description=('Input reserved concurrency. Increase this value if '
'there are steady logs delay despite no errors'))
aes_domain_name = self.node.try_get_context('aes_domain_name')
bucket = f'{aes_domain_name}-{core.Aws.ACCOUNT_ID}'
s3bucket_name_geo = f'{bucket}-geo'
s3bucket_name_log = f'{bucket}-log'
s3bucket_name_snapshot = f'{bucket}-snapshot'
# organizations / multiaccount
org_id = self.node.try_get_context('organizations').get('org_id')
org_mgmt_id = self.node.try_get_context(
'organizations').get('management_id')
org_member_ids = self.node.try_get_context(
'organizations').get('member_ids')
no_org_ids = self.node.try_get_context(
'no_organizations').get('aws_accounts')
# Overwrite default S3 bucket name as customer name
temp_geo = self.node.try_get_context('s3_bucket_name').get('geo')
if temp_geo:
s3bucket_name_geo = temp_geo
else:
print('Using default bucket names')
temp_log = self.node.try_get_context('s3_bucket_name').get('log')
if temp_log:
s3bucket_name_log = temp_log
elif org_id or no_org_ids:
s3bucket_name_log = f'{aes_domain_name}-{self.account}-log'
else:
print('Using default bucket names')
temp_snap = self.node.try_get_context('s3_bucket_name').get('snapshot')
if temp_snap:
s3bucket_name_snapshot = temp_snap
else:
print('Using default bucket names')
kms_cmk_alias = self.node.try_get_context('kms_cmk_alias')
if not kms_cmk_alias:
kms_cmk_alias = 'aes-siem-key'
print('Using default key alais')
######################################################################
# deploy VPC when context is defined as using VPC
######################################################################
# vpc_type is 'new' or 'import' or None
vpc_type = self.node.try_get_context('vpc_type')
if vpc_type == 'new':
is_vpc = True
vpc_cidr = self.node.try_get_context('new_vpc_nw_cidr_block')
subnet_cidr_mask = int(
self.node.try_get_context('new_vpc_subnet_cidr_mask'))
is_vpc = True
# VPC
vpc_aes_siem = aws_ec2.Vpc(
self, 'VpcAesSiem', cidr=vpc_cidr,
max_azs=3, nat_gateways=0,
subnet_configuration=[
aws_ec2.SubnetConfiguration(
subnet_type=aws_ec2.SubnetType.ISOLATED,
name='aes-siem-subnet', cidr_mask=subnet_cidr_mask)])
subnet1 = vpc_aes_siem.isolated_subnets[0]
subnets = [{'subnet_type': aws_ec2.SubnetType.ISOLATED}]
vpc_subnets = aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.ISOLATED)
vpc_aes_siem_opt = vpc_aes_siem.node.default_child.cfn_options
vpc_aes_siem_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
for subnet in vpc_aes_siem.isolated_subnets:
subnet_opt = subnet.node.default_child.cfn_options
subnet_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
elif vpc_type == 'import':
vpc_id = self.node.try_get_context('imported_vpc_id')
vpc_aes_siem = aws_ec2.Vpc.from_lookup(
self, 'VpcAesSiem', vpc_id=vpc_id)
subnet_ids = get_subnet_ids(self)
subnets = []
for number, subnet_id in enumerate(subnet_ids, 1):
obj_id = 'Subenet' + str(number)
subnet = aws_ec2.Subnet.from_subnet_id(self, obj_id, subnet_id)
subnets.append(subnet)
subnet1 = subnets[0]
vpc_subnets = aws_ec2.SubnetSelection(subnets=subnets)
if vpc_type:
is_vpc = True
# Security Group
sg_vpc_noinbound_aes_siem = aws_ec2.SecurityGroup(
self, 'AesSiemVpcNoinboundSecurityGroup',
security_group_name='aes-siem-noinbound-vpc-sg',
vpc=vpc_aes_siem)
sg_vpc_aes_siem = aws_ec2.SecurityGroup(
self, 'AesSiemVpcSecurityGroup',
security_group_name='aes-siem-vpc-sg',
vpc=vpc_aes_siem)
sg_vpc_aes_siem.add_ingress_rule(
peer=aws_ec2.Peer.ipv4(vpc_aes_siem.vpc_cidr_block),
connection=aws_ec2.Port.tcp(443),)
sg_vpc_opt = sg_vpc_aes_siem.node.default_child.cfn_options
sg_vpc_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# VPC Endpoint
vpc_aes_siem.add_gateway_endpoint(
'S3Endpoint', service=aws_ec2.GatewayVpcEndpointAwsService.S3,
subnets=subnets)
vpc_aes_siem.add_interface_endpoint(
'SQSEndpoint', security_groups=[sg_vpc_aes_siem],
service=aws_ec2.InterfaceVpcEndpointAwsService.SQS,)
vpc_aes_siem.add_interface_endpoint(
'KMSEndpoint', security_groups=[sg_vpc_aes_siem],
service=aws_ec2.InterfaceVpcEndpointAwsService.KMS,)
else:
is_vpc = False
is_vpc = core.CfnCondition(
self, 'IsVpc', expression=core.Fn.condition_equals(is_vpc, True))
"""
CloudFormation実行時の条件式の書き方
ClassのBasesが aws_cdk.core.Resource の時は、
node.default_child.cfn_options.condition = is_vpc
ClassのBasesが aws_cdk.core.CfnResource の時は、
cfn_options.condition = is_vpc
"""
######################################################################
# create cmk of KMS to encrypt S3 bucket
######################################################################
kms_aes_siem = aws_kms.Key(
self, 'KmsAesSiemLog', description='CMK for SIEM solution',
removal_policy=core.RemovalPolicy.RETAIN)
aws_kms.Alias(
self, 'KmsAesSiemLogAlias', alias_name=kms_cmk_alias,
target_key=kms_aes_siem,
removal_policy=core.RemovalPolicy.RETAIN)
kms_aes_siem.add_to_resource_policy(
aws_iam.PolicyStatement(
sid='Allow GuardDuty to use the key',
actions=['kms:GenerateDataKey'],
principals=[aws_iam.ServicePrincipal(
'guardduty.amazonaws.com')],
resources=['*'],),)
kms_aes_siem.add_to_resource_policy(
aws_iam.PolicyStatement(
sid='Allow VPC Flow Logs to use the key',
actions=['kms:Encrypt', 'kms:Decrypt', 'kms:ReEncrypt*',
'kms:GenerateDataKey*', 'kms:DescribeKey'],
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
resources=['*'],),)
# basic policy
key_policy_basic1 = aws_iam.PolicyStatement(
sid='Allow principals in the account to decrypt log files',
actions=['kms:DescribeKey', 'kms:ReEncryptFrom'],
principals=[aws_iam.AccountPrincipal(
account_id=core.Aws.ACCOUNT_ID)],
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_basic1)
# for Athena
key_policy_athena = aws_iam.PolicyStatement(
sid='Allow Athena to query s3 objects with this key',
actions=['kms:Decrypt', 'kms:DescribeKey', 'kms:Encrypt',
'kms:GenerateDataKey*', 'kms:ReEncrypt*'],
principals=[aws_iam.AccountPrincipal(
account_id=core.Aws.ACCOUNT_ID)],
resources=['*'],
conditions={'ForAnyValue:StringEquals': {
'aws:CalledVia': 'athena.amazonaws.com'}})
kms_aes_siem.add_to_resource_policy(key_policy_athena)
# for CloudTrail
key_policy_trail1 = aws_iam.PolicyStatement(
sid='Allow CloudTrail to describe key',
actions=['kms:DescribeKey'],
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_trail1)
key_policy_trail2 = aws_iam.PolicyStatement(
sid=('Allow CloudTrail to encrypt logs'),
actions=['kms:GenerateDataKey*'],
principals=[aws_iam.ServicePrincipal(
'cloudtrail.amazonaws.com')],
resources=['*'],
conditions={'StringLike': {
'kms:EncryptionContext:aws:cloudtrail:arn': [
f'arn:aws:cloudtrail:*:{core.Aws.ACCOUNT_ID}:trail/*']}})
kms_aes_siem.add_to_resource_policy(key_policy_trail2)
######################################################################
# create s3 bucket
######################################################################
block_pub = aws_s3.BlockPublicAccess(
block_public_acls=True,
ignore_public_acls=True,
block_public_policy=True,
restrict_public_buckets=True
)
s3_geo = aws_s3.Bucket(
self, 'S3BucketForGeoip', block_public_access=block_pub,
bucket_name=s3bucket_name_geo,
# removal_policy=core.RemovalPolicy.DESTROY,
)
# create s3 bucket for log collector
s3_log = aws_s3.Bucket(
self, 'S3BucketForLog', block_public_access=block_pub,
bucket_name=s3bucket_name_log, versioned=True,
encryption=aws_s3.BucketEncryption.S3_MANAGED,
# removal_policy=core.RemovalPolicy.DESTROY,
)
# create s3 bucket for aes snapshot
s3_snapshot = aws_s3.Bucket(
self, 'S3BucketForSnapshot', block_public_access=block_pub,
bucket_name=s3bucket_name_snapshot,
# removal_policy=core.RemovalPolicy.DESTROY,
)
######################################################################
# IAM Role
######################################################################
# delopyment policy for lambda deploy-aes
arn_prefix = f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}'
loggroup_aes = f'log-group:/aws/aes/domains/{aes_domain_name}/*'
loggroup_opensearch = (
f'log-group:/aws/OpenSearchService/domains/{aes_domain_name}/*')
loggroup_lambda = 'log-group:/aws/lambda/aes-siem-*'
policydoc_create_loggroup = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=[
'logs:PutResourcePolicy',
'logs:DescribeLogGroups',
'logs:DescribeLogStreams'
],
resources=[f'{arn_prefix}:*', ]
),
aws_iam.PolicyStatement(
actions=[
'logs:CreateLogGroup', 'logs:CreateLogStream',
'logs:PutLogEvents', 'logs:PutRetentionPolicy'],
resources=[
f'{arn_prefix}:{loggroup_aes}',
f'{arn_prefix}:{loggroup_opensearch}',
f'{arn_prefix}:{loggroup_lambda}',
],
)
]
)
policydoc_crhelper = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=[
'lambda:AddPermission',
'lambda:RemovePermission',
'events:ListRules',
'events:PutRule',
'events:DeleteRule',
'events:PutTargets',
'events:RemoveTargets'],
resources=['*']
)
]
)
# snaphot rule for AES
policydoc_snapshot = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=['s3:ListBucket'],
resources=[s3_snapshot.bucket_arn]
),
aws_iam.PolicyStatement(
actions=['s3:GetObject', 's3:PutObject',
's3:DeleteObject'],
resources=[s3_snapshot.bucket_arn + '/*']
)
]
)
aes_siem_snapshot_role = aws_iam.Role(
self, 'AesSiemSnapshotRole',
role_name='aes-siem-snapshot-role',
inline_policies=[policydoc_snapshot, ],
assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com')
)
policydoc_assume_snapshrole = aws_iam.PolicyDocument(
statements=[
aws_iam.PolicyStatement(
actions=['iam:PassRole'],
resources=[aes_siem_snapshot_role.role_arn]
),
]
)
aes_siem_deploy_role_for_lambda = aws_iam.Role(
self, 'AesSiemDeployRoleForLambda',
role_name='aes-siem-deploy-role-for-lambda',
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonOpenSearchServiceFullAccess'),
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
],
inline_policies=[policydoc_assume_snapshrole, policydoc_snapshot,
policydoc_create_loggroup, policydoc_crhelper],
assumed_by=aws_iam.ServicePrincipal('lambda.amazonaws.com')
)
if vpc_type:
aes_siem_deploy_role_for_lambda.add_managed_policy(
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaVPCAccessExecutionRole')
)
# for alert from Amazon OpenSearch Service
aes_siem_sns_role = aws_iam.Role(
self, 'AesSiemSnsRole',
role_name='aes-siem-sns-role',
assumed_by=aws_iam.ServicePrincipal('es.amazonaws.com')
)
# EC2 role
aes_siem_es_loader_ec2_role = aws_iam.Role(
self, 'AesSiemEsLoaderEC2Role',
role_name='aes-siem-es-loader-for-ec2',
assumed_by=aws_iam.ServicePrincipal('ec2.amazonaws.com'),
)
aws_iam.CfnInstanceProfile(
self, 'AesSiemEsLoaderEC2InstanceProfile',
instance_profile_name=aes_siem_es_loader_ec2_role.role_name,
roles=[aes_siem_es_loader_ec2_role.role_name]
)
######################################################################
# in VPC
######################################################################
aes_role_exist = check_iam_role('/aws-service-role/es.amazonaws.com/')
if vpc_type and not aes_role_exist:
slr_aes = aws_iam.CfnServiceLinkedRole(
self, 'AWSServiceRoleForAmazonOpenSearchService',
aws_service_name='es.amazonaws.com',
description='Created by cloudformation of siem stack'
)
slr_aes.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
######################################################################
# SQS for es-laoder's DLQ
######################################################################
sqs_aes_siem_dlq = aws_sqs.Queue(
self, 'AesSiemDlq', queue_name='aes-siem-dlq',
retention_period=core.Duration.days(14))
sqs_aes_siem_splitted_logs = aws_sqs.Queue(
self, 'AesSiemSqsSplitLogs',
queue_name='aes-siem-sqs-splitted-logs',
dead_letter_queue=aws_sqs.DeadLetterQueue(
max_receive_count=2, queue=sqs_aes_siem_dlq),
visibility_timeout=core.Duration.seconds(ES_LOADER_TIMEOUT),
retention_period=core.Duration.days(14))
######################################################################
# Setup Lambda
######################################################################
# setup lambda of es_loader
lambda_es_loader_vpc_kwargs = {}
if vpc_type:
lambda_es_loader_vpc_kwargs = {
'security_group': sg_vpc_noinbound_aes_siem,
'vpc': vpc_aes_siem,
'vpc_subnets': vpc_subnets,
}
lambda_es_loader = aws_lambda.Function(
self, 'LambdaEsLoader', **lambda_es_loader_vpc_kwargs,
function_name='aes-siem-es-loader',
description=f'{SOLUTION_NAME} / es-loader',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
# code=aws_lambda.Code.asset('../lambda/es_loader.zip'),
code=aws_lambda.Code.asset('../lambda/es_loader'),
handler='index.lambda_handler',
memory_size=2048,
timeout=core.Duration.seconds(ES_LOADER_TIMEOUT),
reserved_concurrent_executions=(
reserved_concurrency.value_as_number),
dead_letter_queue_enabled=True,
dead_letter_queue=sqs_aes_siem_dlq,
environment={
'GEOIP_BUCKET': s3bucket_name_geo, 'LOG_LEVEL': 'info',
'POWERTOOLS_LOGGER_LOG_EVENT': 'false',
'POWERTOOLS_SERVICE_NAME': 'es-loader',
'POWERTOOLS_METRICS_NAMESPACE': 'SIEM'})
es_loader_newver = lambda_es_loader.add_version(
name=__version__, description=__version__)
es_loader_opt = es_loader_newver.node.default_child.cfn_options
es_loader_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# send only
# sqs_aes_siem_dlq.grant(lambda_es_loader, 'sqs:SendMessage')
# send and reieve. but it must be loop
sqs_aes_siem_dlq.grant(
lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage',
'sqs:DeleteMessage', 'sqs:GetQueueAttributes')
sqs_aes_siem_splitted_logs.grant(
lambda_es_loader, 'sqs:SendMessage', 'sqs:ReceiveMessage',
'sqs:DeleteMessage', 'sqs:GetQueueAttributes')
lambda_es_loader.add_event_source(
aws_lambda_event_sources.SqsEventSource(
sqs_aes_siem_splitted_logs, batch_size=1))
# es-loaer on EC2 role
sqs_aes_siem_dlq.grant(
aes_siem_es_loader_ec2_role, 'sqs:GetQueue*', 'sqs:ListQueues*',
'sqs:ReceiveMessage*', 'sqs:DeleteMessage*')
lambda_geo = aws_lambda.Function(
self, 'LambdaGeoipDownloader',
function_name='aes-siem-geoip-downloader',
description=f'{SOLUTION_NAME} / geoip-downloader',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
code=aws_lambda.Code.asset('../lambda/geoip_downloader'),
handler='index.lambda_handler',
memory_size=320,
timeout=core.Duration.seconds(300),
environment={
's3bucket_name': s3bucket_name_geo,
'license_key': geoip_license_key.value_as_string,
}
)
lambda_geo_newver = lambda_geo.add_version(
name=__version__, description=__version__)
lamba_geo_opt = lambda_geo_newver.node.default_child.cfn_options
lamba_geo_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
######################################################################
# setup OpenSearch Service
######################################################################
lambda_deploy_es = aws_lambda.Function(
self, 'LambdaDeployAES',
function_name='aes-siem-deploy-aes',
description=f'{SOLUTION_NAME} / opensearch domain deployment',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
# code=aws_lambda.Code.asset('../lambda/deploy_es.zip'),
code=aws_lambda.Code.asset('../lambda/deploy_es'),
handler='index.aes_domain_handler',
memory_size=128,
timeout=core.Duration.seconds(300),
environment={
'accountid': core.Aws.ACCOUNT_ID,
'aes_domain_name': aes_domain_name,
'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn,
'es_loader_role': lambda_es_loader.role.role_arn,
'allow_source_address': allow_source_address.value_as_string,
},
role=aes_siem_deploy_role_for_lambda,
)
lambda_deploy_es.add_environment(
's3_snapshot', s3_snapshot.bucket_name)
if vpc_type:
lambda_deploy_es.add_environment(
'vpc_subnet_id', subnet1.subnet_id)
lambda_deploy_es.add_environment(
'security_group_id', sg_vpc_aes_siem.security_group_id)
else:
lambda_deploy_es.add_environment('vpc_subnet_id', 'None')
lambda_deploy_es.add_environment('security_group_id', 'None')
deploy_es_newver = lambda_deploy_es.add_version(
name=__version__, description=__version__)
deploy_es_opt = deploy_es_newver.node.default_child.cfn_options
deploy_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
# execute lambda_deploy_es to deploy Amaozon ES Domain
aes_domain = aws_cloudformation.CfnCustomResource(
self, 'AesSiemDomainDeployedR2',
service_token=lambda_deploy_es.function_arn,)
aes_domain.add_override('Properties.ConfigVersion', __version__)
es_endpoint = aes_domain.get_att('es_endpoint').to_string()
lambda_es_loader.add_environment('ES_ENDPOINT', es_endpoint)
lambda_es_loader.add_environment(
'SQS_SPLITTED_LOGS_URL', sqs_aes_siem_splitted_logs.queue_url)
lambda_configure_es_vpc_kwargs = {}
if vpc_type:
lambda_configure_es_vpc_kwargs = {
'security_group': sg_vpc_noinbound_aes_siem,
'vpc': vpc_aes_siem,
'vpc_subnets': aws_ec2.SubnetSelection(subnets=[subnet1, ]), }
lambda_configure_es = aws_lambda.Function(
self, 'LambdaConfigureAES', **lambda_configure_es_vpc_kwargs,
function_name='aes-siem-configure-aes',
description=f'{SOLUTION_NAME} / opensearch configuration',
runtime=aws_lambda.Runtime.PYTHON_3_8,
architecture=aws_lambda.Architecture.X86_64,
# architecture=region_mapping.find_in_map(
# core.Aws.REGION, 'LambdaArm'),
code=aws_lambda.Code.asset('../lambda/deploy_es'),
handler='index.aes_config_handler',
memory_size=128,
timeout=core.Duration.seconds(300),
environment={
'accountid': core.Aws.ACCOUNT_ID,
'aes_domain_name': aes_domain_name,
'aes_admin_role': aes_siem_deploy_role_for_lambda.role_arn,
'es_loader_role': lambda_es_loader.role.role_arn,
'allow_source_address': allow_source_address.value_as_string,
'es_endpoint': es_endpoint,
},
role=aes_siem_deploy_role_for_lambda,
)
lambda_configure_es.add_environment(
's3_snapshot', s3_snapshot.bucket_name)
if vpc_type:
lambda_configure_es.add_environment(
'vpc_subnet_id', subnet1.subnet_id)
lambda_configure_es.add_environment(
'security_group_id', sg_vpc_aes_siem.security_group_id)
else:
lambda_configure_es.add_environment('vpc_subnet_id', 'None')
lambda_configure_es.add_environment('security_group_id', 'None')
configure_es_newver = lambda_configure_es.add_version(
name=__version__, description=__version__)
configure_es_opt = configure_es_newver.node.default_child.cfn_options
configure_es_opt.deletion_policy = core.CfnDeletionPolicy.RETAIN
aes_config = aws_cloudformation.CfnCustomResource(
self, 'AesSiemDomainConfiguredR2',
service_token=lambda_configure_es.function_arn,)
aes_config.add_override('Properties.ConfigVersion', __version__)
aes_config.add_depends_on(aes_domain)
aes_config.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
es_arn = (f'arn:aws:es:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}'
f':domain/{aes_domain_name}')
# grant permission to es_loader role
inline_policy_to_load_entries_into_es = aws_iam.Policy(
self, 'aes-siem-policy-to-load-entries-to-es',
policy_name='aes-siem-policy-to-load-entries-to-es',
statements=[
aws_iam.PolicyStatement(
actions=['es:*'],
resources=[es_arn + '/*', ]),
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_to_load_entries_into_es)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_to_load_entries_into_es)
# grant additional permission to es_loader role
additional_kms_cmks = self.node.try_get_context('additional_kms_cmks')
if additional_kms_cmks:
inline_policy_access_to_additional_cmks = aws_iam.Policy(
self, 'access_to_additional_cmks',
policy_name='access_to_additional_cmks',
statements=[
aws_iam.PolicyStatement(
actions=['kms:Decrypt'],
resources=sorted(set(additional_kms_cmks))
)
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_access_to_additional_cmks)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_access_to_additional_cmks)
additional_buckets = self.node.try_get_context('additional_s3_buckets')
if additional_buckets:
buckets_list = []
for bucket in additional_buckets:
buckets_list.append(f'arn:aws:s3:::{bucket}')
buckets_list.append(f'arn:aws:s3:::{bucket}/*')
inline_policy_access_to_additional_buckets = aws_iam.Policy(
self, 'access_to_additional_buckets',
policy_name='access_to_additional_buckets',
statements=[
aws_iam.PolicyStatement(
actions=['s3:GetObject*', 's3:GetBucket*', 's3:List*'],
resources=sorted(set(buckets_list))
)
]
)
lambda_es_loader.role.attach_inline_policy(
inline_policy_access_to_additional_buckets)
aes_siem_es_loader_ec2_role.attach_inline_policy(
inline_policy_access_to_additional_buckets)
kms_aes_siem.grant_decrypt(lambda_es_loader)
kms_aes_siem.grant_decrypt(aes_siem_es_loader_ec2_role)
######################################################################
# s3 notification and grant permisssion
######################################################################
s3_geo.grant_read_write(lambda_geo)
s3_geo.grant_read(lambda_es_loader)
s3_geo.grant_read(aes_siem_es_loader_ec2_role)
s3_log.grant_read(lambda_es_loader)
s3_log.grant_read(aes_siem_es_loader_ec2_role)
# create s3 notification for es_loader
notification = aws_s3_notifications.LambdaDestination(lambda_es_loader)
# assign notification for the s3 PUT event type
# most log system use PUT, but also CLB use POST & Multipart Upload
s3_log.add_event_notification(
aws_s3.EventType.OBJECT_CREATED, notification,
aws_s3.NotificationKeyFilter(prefix='AWSLogs/'))
# For user logs, not AWS logs
s3_log.add_event_notification(
aws_s3.EventType.OBJECT_CREATED, notification,
aws_s3.NotificationKeyFilter(prefix='UserLogs/'))
# Download geoip to S3 once by executing lambda_geo
get_geodb = aws_cloudformation.CfnCustomResource(
self, 'ExecLambdaGeoipDownloader',
service_token=lambda_geo.function_arn,)
get_geodb.cfn_options.deletion_policy = core.CfnDeletionPolicy.RETAIN
# Download geoip every day at 6PM UTC
rule = aws_events.Rule(
self, 'CwlRuleLambdaGeoipDownloaderDilly',
schedule=aws_events.Schedule.rate(core.Duration.hours(12)))
rule.add_target(aws_events_targets.LambdaFunction(lambda_geo))
######################################################################
# bucket policy
######################################################################
s3_awspath = s3_log.bucket_arn + '/AWSLogs/' + core.Aws.ACCOUNT_ID
bucket_policy_common1 = aws_iam.PolicyStatement(
sid='ELB Policy',
principals=[aws_iam.AccountPrincipal(
account_id=region_mapping.find_in_map(
core.Aws.REGION, 'ElbV2AccountId'))],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],)
# NLB / ALB / R53resolver / VPC Flow Logs
bucket_policy_elb1 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryAclCheck For ALB NLB R53Resolver Flowlogs',
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
actions=['s3:GetBucketAcl', 's3:ListBucket'],
resources=[s3_log.bucket_arn],)
bucket_policy_elb2 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryWrite For ALB NLB R53Resolver Flowlogs',
principals=[aws_iam.ServicePrincipal(
'delivery.logs.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_common1)
s3_log.add_to_resource_policy(bucket_policy_elb1)
s3_log.add_to_resource_policy(bucket_policy_elb2)
# CloudTrail
bucket_policy_trail1 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryAclCheck For Cloudtrail',
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:GetBucketAcl'], resources=[s3_log.bucket_arn],)
bucket_policy_trail2 = aws_iam.PolicyStatement(
sid='AWSLogDeliveryWrite For CloudTrail',
principals=[aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_trail1)
s3_log.add_to_resource_policy(bucket_policy_trail2)
# GuardDuty
bucket_policy_gd1 = aws_iam.PolicyStatement(
sid='Allow GuardDuty to use the getBucketLocation operation',
principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')],
actions=['s3:GetBucketLocation'], resources=[s3_log.bucket_arn],)
bucket_policy_gd2 = aws_iam.PolicyStatement(
sid='Allow GuardDuty to upload objects to the bucket',
principals=[aws_iam.ServicePrincipal('guardduty.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_log.bucket_arn + '/*'],)
bucket_policy_gd5 = aws_iam.PolicyStatement(
sid='Deny non-HTTPS access', effect=aws_iam.Effect.DENY,
actions=['s3:*'], resources=[s3_log.bucket_arn + '/*'],
conditions={'Bool': {'aws:SecureTransport': 'false'}})
bucket_policy_gd5.add_any_principal()
s3_log.add_to_resource_policy(bucket_policy_gd1)
s3_log.add_to_resource_policy(bucket_policy_gd2)
s3_log.add_to_resource_policy(bucket_policy_gd5)
# Config
bucket_policy_config1 = aws_iam.PolicyStatement(
sid='AWSConfig BucketPermissionsCheck and BucketExistenceCheck',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:GetBucketAcl', 's3:ListBucket'],
resources=[s3_log.bucket_arn],)
bucket_policy_config2 = aws_iam.PolicyStatement(
sid='AWSConfigBucketDelivery',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:PutObject'], resources=[s3_awspath + '/Config/*'],
conditions={
'StringEquals': {'s3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_config1)
s3_log.add_to_resource_policy(bucket_policy_config2)
# geoip
bucket_policy_geo1 = aws_iam.PolicyStatement(
sid='Allow geoip downloader and es-loader to read/write',
principals=[lambda_es_loader.role, lambda_geo.role],
actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'],
resources=[s3_geo.bucket_arn + '/*'],)
s3_geo.add_to_resource_policy(bucket_policy_geo1)
# ES Snapshot
bucket_policy_snapshot = aws_iam.PolicyStatement(
sid='Allow ES to store snapshot',
principals=[aes_siem_snapshot_role],
actions=['s3:PutObject', 's3:GetObject', 's3:DeleteObject'],
resources=[s3_snapshot.bucket_arn + '/*'],)
s3_snapshot.add_to_resource_policy(bucket_policy_snapshot)
######################################################################
# for multiaccount / organizaitons
######################################################################
if org_id or no_org_ids:
##################################################################
# KMS key policy for multiaccount / organizaitons
##################################################################
# for CloudTrail
cond_tail2 = self.make_resource_list(
path='arn:aws:cloudtrail:*:', tail=':trail/*',
keys=self.list_without_none(org_mgmt_id, no_org_ids))
key_policy_mul_trail2 = aws_iam.PolicyStatement(
sid=('Allow CloudTrail to encrypt logs for multiaccounts'),
actions=['kms:GenerateDataKey*'],
principals=[aws_iam.ServicePrincipal(
'cloudtrail.amazonaws.com')],
resources=['*'],
conditions={'StringLike': {
'kms:EncryptionContext:aws:cloudtrail:arn': cond_tail2}})
kms_aes_siem.add_to_resource_policy(key_policy_mul_trail2)
# for replicaiton
key_policy_rep1 = aws_iam.PolicyStatement(
sid=('Enable cross account encrypt access for S3 Cross Region '
'Replication'),
actions=['kms:Encrypt'],
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
resources=['*'],)
kms_aes_siem.add_to_resource_policy(key_policy_rep1)
##################################################################
# Buckdet Policy for multiaccount / organizaitons
##################################################################
s3_log_bucket_arn = 'arn:aws:s3:::' + s3bucket_name_log
# for CloudTrail
s3_mulpaths = self.make_resource_list(
path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/*',
keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids))
bucket_policy_org_trail = aws_iam.PolicyStatement(
sid='AWSCloudTrailWrite for Multiaccounts / Organizations',
principals=[
aws_iam.ServicePrincipal('cloudtrail.amazonaws.com')],
actions=['s3:PutObject'], resources=s3_mulpaths,
conditions={'StringEquals': {
's3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_org_trail)
# config
s3_conf_multpaths = self.make_resource_list(
path=f'{s3_log_bucket_arn}/AWSLogs/', tail='/Config/*',
keys=self.list_without_none(org_id, org_mgmt_id, no_org_ids))
bucket_policy_mul_config2 = aws_iam.PolicyStatement(
sid='AWSConfigBucketDelivery',
principals=[aws_iam.ServicePrincipal('config.amazonaws.com')],
actions=['s3:PutObject'], resources=s3_conf_multpaths,
conditions={'StringEquals': {
's3:x-amz-acl': 'bucket-owner-full-control'}})
s3_log.add_to_resource_policy(bucket_policy_mul_config2)
# for replication
bucket_policy_rep1 = aws_iam.PolicyStatement(
sid='PolicyForDestinationBucket / Permissions on objects',
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
actions=['s3:ReplicateDelete', 's3:ReplicateObject',
's3:ReplicateTags', 's3:GetObjectVersionTagging',
's3:ObjectOwnerOverrideToBucketOwner'],
resources=[f'{s3_log_bucket_arn}/*'])
bucket_policy_rep2 = aws_iam.PolicyStatement(
sid='PolicyForDestinationBucket / Permissions on bucket',
principals=self.make_account_principals(
org_mgmt_id, org_member_ids, no_org_ids),
actions=['s3:List*', 's3:GetBucketVersioning',
's3:PutBucketVersioning'],
resources=[f'{s3_log_bucket_arn}'])
s3_log.add_to_resource_policy(bucket_policy_rep1)
s3_log.add_to_resource_policy(bucket_policy_rep2)
######################################################################
# SNS topic for Amazon OpenSearch Service Alert
######################################################################
sns_topic = aws_sns.Topic(
self, 'SnsTopic', topic_name='aes-siem-alert',
display_name='AES SIEM')
sns_topic.add_subscription(aws_sns_subscriptions.EmailSubscription(
email_address=sns_email.value_as_string))
sns_topic.grant_publish(aes_siem_sns_role)
######################################################################
# output of CFn
######################################################################
kibanaurl = f'https://{es_endpoint}/_dashboards/'
kibanaadmin = aes_domain.get_att('kibanaadmin').to_string()
kibanapass = aes_domain.get_att('kibanapass').to_string()
core.CfnOutput(self, 'RoleDeploy', export_name='role-deploy',
value=aes_siem_deploy_role_for_lambda.role_arn)
core.CfnOutput(self, 'DashboardsUrl', export_name='dashboards-url',
value=kibanaurl)
core.CfnOutput(self, 'DashboardsPassword',
export_name='dashboards-pass', value=kibanapass,
description=('Please change the password in OpenSearch '
'Dashboards ASAP'))
core.CfnOutput(self, 'DashboardsAdminID',
export_name='dashboards-admin', value=kibanaadmin)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Tag all constructs with the project for easy billing drilldown,
# filtering, and organization.
core.Tags.of(self).add('project', 'MediaTranscription')
# Media files bucket
media_bucket = s3.Bucket(
self,
'media-transcription-bucket',
encryption=s3.BucketEncryption.S3_MANAGED,
)
# SQS queue for media files bucket event notifications
media_bucket_event_queue = sqs.Queue(
self,
'media-transcription-event-notification-queue',
queue_name='media-transcription-event-notification-queue',
visibility_timeout=core.Duration.seconds(60),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=3,
queue=sqs.Queue(
self,
'media-transcription-event-notifications-dlq',
queue_name='media-transcription-event-notifications-dlq',
)),
)
# S3 object created notifications sent to SQS queue
media_bucket.add_event_notification(
s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(media_bucket_event_queue),
*[s3.NotificationKeyFilter(prefix='media-input/')],
)
# Lambda function to create/submit Transcribe jobs
transcribe_job_init_fn = lambda_.Function(
self,
'transcribe-job-init-fn',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.from_asset(
'../lambdas/transcribe-job-init-fn',
# The following is just dumb.
# The Lambda runtime doesn't use the latest boto3 by default.
# In order to use the latest boto3, we have to pip install
# and bundle locally using Docker.
# Q: Why need the latest boto3?
# A: https://github.com/boto/boto3/issues/2630
# I'll have to delete the ECR containers to avoid cost.
# TODO: Revert back to normal in like a month I guess.
bundling={
'image':
lambda_.Runtime.PYTHON_3_8.bundling_docker_image,
'command': [
'bash', '-c',
'\n pip install -r requirements.txt -t /asset-output &&\n cp -au . /asset-output\n '
]
}),
handler='fn.handler',
reserved_concurrent_executions=1, # Effectively single-threaded
)
# Triggered by SQS messages created for media file puts
transcribe_job_init_fn.add_event_source(
les.SqsEventSource(
queue=media_bucket_event_queue,
batch_size=5,
enabled=True,
))
# Grant access to start transcription jobs
transcribe_job_init_fn.add_to_role_policy(
statement=iam.PolicyStatement(
actions=[
'transcribe:StartTranscriptionJob',
],
resources=['*'],
effect=iam.Effect.ALLOW,
))
# Grant Lambda role to read and write to input and output portions of
# the S3 bucket.
# Q: Why grant Lambda the permissions instead of Transcribe service?
# A: Two-fold:
# - i) https://amzn.to/321Nx5I
# - ii) Granting just to this Lambda means other Transcribe jobs
# across the account cannot use this bucket (least privilege).
media_bucket.grant_read(
identity=transcribe_job_init_fn.grant_principal,
objects_key_pattern='media-input/*')
# Cannot specify a prefix for writes as Transcribe will not accept
# a job unless it has write permission on the whole bucket.
# Edit: The above statement was when I had to use '*' for writes. But
# now, I granted access to that .write_access_check_file.temp
# file and it seems to all work now?
media_bucket.grant_write(
identity=transcribe_job_init_fn.grant_principal,
objects_key_pattern='transcribe-output-raw/*')
# This is just as frustrating to you as it is to me.
media_bucket.grant_write(
identity=transcribe_job_init_fn.grant_principal,
objects_key_pattern='.write_access_check_file.temp')
# DynamoDB table for Jobs metadata
jobs_metadata_table = ddb.Table(
self,
'MediaTranscription-TranscriptionJobs',
table_name='MediaTranscription-TranscriptionJobs',
partition_key=ddb.Attribute(
name='Bucket-Key-ETag',
type=ddb.AttributeType.STRING,
),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST,
)
jobs_metadata_table.grant(transcribe_job_init_fn.grant_principal, *[
'dynamodb:GetItem',
'dynamodb:PutItem',
])
# Create IAM Group with read/write permissions to S3 bucket
# TODO: Make this more federated and robust
console_users_group = iam.Group(self, 'MediaTranscriptionConsoleUsers')
console_users_group.attach_inline_policy(policy=iam.Policy(
self,
'MediaTranscriptionConsoleUserS3Access',
statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
's3:ListBucket',
],
resources=[
media_bucket.bucket_arn,
],
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
's3:GetObject',
's3:PutObject',
],
resources=[
media_bucket.arn_for_objects('media-input/*'),
],
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
's3:GetObject',
],
resources=[
media_bucket.arn_for_objects(
'transcribe-output-raw/*'),
],
),
],
))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
layer = aws.PipLayers(self,
"scoreboard_layer",
layers={
"htmlgen": "htmlgen/requirements.txt",
"parse_globals":
"parse_globals/requirements.txt",
"bot": "bot/requirements.txt"
})
# Create
# * the generator function
# * Namemap-table
# * allow generator to read from namemap-table
# (This might change - why not pass the mapping structure in the message?)
# * Datacache-bucket
# * Allow generator to read and write to bucket
htmlgen = aws.Function(self,
"htmlgen",
layers=layer.layers,
timeout=core.Duration.seconds(20),
memory_size=1024)
# id: str (boardid), name: str (username), value: str (replacement value)
namemap = aws.Table(self,
"namemap",
sort_key=aws_dynamodb.Attribute(
name='name',
type=aws_dynamodb.AttributeType.STRING),
removal_policy=CONFIGDATA)
namemap.grant_read_data(htmlgen)
no_point_days = aws.Table(self, "nopointdays")
# id: str (boardid), day: int, results_1: dict ({player: score, ...}), results_2: dict ({player: score, ...})
globalscores = aws.Table(self,
"globalscores",
partition_key=aws_dynamodb.Attribute(
name='year',
type=aws_dynamodb.AttributeType.NUMBER),
sort_key=aws_dynamodb.Attribute(
name='day',
type=aws_dynamodb.AttributeType.NUMBER),
removal_policy=EPHEMERALDATA)
parse_globals = aws.Function(self,
"parse_globals",
layers=layer.layers,
timeout=core.Duration.seconds(20),
memory_size=1024)
parse_globals.add_environment("DDB_GLOBALSCORES",
globalscores.table_name)
globalscores.grant_read_write_data(parse_globals)
globalscores.grant_read_data(htmlgen)
timestamps = aws.Table(self,
"timestamps",
removal_policy=EPHEMERALDATA)
htmlgen.add_environment("DDB_TIMESTAMPS", timestamps.table_name)
timestamps.grant_write_data(htmlgen)
datacache = aws.Bucket(self, "datacache")
datacache.grant_read_write(htmlgen)
htmlbucket = aws.Bucket(
self,
"html",
removal_policy=EPHEMERALDATA,
auto_delete_objects=True,
block_public_access=None,
website_error_document="error.html",
website_index_document="scoreboard.html",
cors=[
aws_s3.CorsRule(allowed_methods=[aws_s3.HttpMethods.GET],
allowed_headers=["*"],
allowed_origins=["*"])
])
htmlbucket.grant_public_access()
core.CfnOutput(self,
f"{id}_bucketurl",
value=f"BUCKET_URL={htmlbucket.bucket_website_url}")
htmlbucket.grant_read_write(htmlgen)
htmlgen.add_environment("S3_DATACACHE", datacache.bucket_name)
htmlgen.add_environment("S3_HTML", htmlbucket.bucket_name)
htmlgen.add_environment("DDB_NAMEMAP", namemap.table_name)
aws_s3_deployment.BucketDeployment(
self,
"StaticHtml",
sources=[aws_s3_deployment.Source.asset("htmlgen/frontend")],
destination_bucket=htmlbucket,
prune=False)
# Create
# * spawner function
# * boardconfig-table
# * allow spawner to read from boardconfig-table
# * generator_queue
# allow spawner to post messages to queue
spawner = aws.Function(self, "spawner", layers=layer.layers)
boardconfig = aws.Table(
self,
"boardconfig",
stream=aws_dynamodb.StreamViewType.NEW_AND_OLD_IMAGES,
removal_policy=CONFIGDATA)
boardconfig.grant_read_data(spawner)
spawner.add_environment("DDB_CONFIG", boardconfig.table_name)
spawner.add_environment("DDB_NOPOINTDAYS", no_point_days.table_name)
boardconfig_source = aws_lambda_event_sources.DynamoEventSource(
boardconfig, starting_position=aws_lambda.StartingPosition.LATEST)
boarddeletions = aws.Function(self, "boarddeletions")
boarddeletions.add_event_source(boardconfig_source)
boarddeletions.add_environment("S3_HTML", htmlbucket.bucket_name)
htmlbucket.grant_read_write(boarddeletions)
generator_queue = aws.Queue(self, "generator_queue")
generator_queue.grant_send_messages(spawner)
spawner.add_environment("SQS_GENERATOR", generator_queue.queue_name)
spawner.add_environment("DDB_TIMESTAMPS", timestamps.table_name)
timestamps.grant_read_data(spawner)
# Connect the generator_queue to the htmlgen-function
event_source = aws_lambda_event_sources.SqsEventSource(generator_queue,
batch_size=10)
htmlgen.add_event_source(event_source)
# Admin API
adminhandler = aws.Function(self, "adminhandler")
adminhandlerApi = aws_apigateway.LambdaRestApi(self,
"adminapi",
handler=adminhandler)
core.CfnOutput(self,
"root_url",
value=f"Admin URL={adminhandlerApi.url_for_path()}")
adminhandler.add_environment("DDB_CONFIG", boardconfig.table_name)
boardconfig.grant_read_write_data(adminhandler)
# Slack API
api = aws.RestApi(self, "slack")
slack = aws.ResourceWithLambda(
self,
"bot",
verb="POST",
description="Handle incoming Slack-bot interaction",
parent_resource=api.root,
lambda_layers=[layer.idlayers["bot"]])
slack.handler.add_environment(
"BOT_TOKEN", read_token_from_file('slack_bot_token.txt'))
slack.handler.add_environment(
"BOT_VERIFICATION",
read_token_from_file('slack_verification_token.txt'))
# "xoxb-1033954193568-1654676166455-Vzom9aQY9NUjAYR5mhKZP70k")
slack.handler.add_environment("DDB_CONFIG", boardconfig.table_name)
slack.handler.add_environment("DDB_NAMEMAP", namemap.table_name)
namemap.grant_read_write_data(slack.handler)
boardconfig.grant_read_write_data(slack.handler)
# aws.Rule(
# self,
# "Test",
# description="Remove after functions verified - Fire every minute for some duration in Februaryx",
# schedule=aws_events.Schedule.cron(minute="*", hour="*", week_day="2", month="FEB"),
# target=spawner)
aws.Rule(self,
"RestOfYear",
description="Fire every week jan-novx",
schedule=aws_events.Schedule.cron(minute="0",
hour="4",
week_day="2",
month="JAN-NOV"),
target=spawner)
aws.Rule(self,
"Mornings_December",
description="Every second minute 06-08 (CET) 1-25 decx",
schedule=aws_events.Schedule.cron(minute="0/2",
hour="6-7",
day="1-25",
month="DEC"),
target=spawner)
aws.Rule(self,
"Daytime_December",
description="Every 20 minutes 08-15 (CET) 1-25 decx",
schedule=aws_events.Schedule.cron(minute="0/20",
hour="8-15",
day="1-25",
month="DEC"),
target=spawner)
aws.Rule(self,
"Nighttime_December",
description="Every hour 00-6,14-24 (CET) 1-25 decx",
schedule=aws_events.Schedule.cron(minute="0",
hour="0-6,14-23",
day="1-25",
month="DEC"),
target=spawner)
aws.Rule(self,
"EndOf_December",
description="Every hour 9-23 (CET) 25-31 decx",
schedule=aws_events.Schedule.cron(minute="0",
hour="9-23",
day="26-31",
month="DEC"),
target=spawner)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
self.current_dir = os.path.dirname(__file__)
self.bucket = s3.Bucket(
self,
"qs-migration-bucket",
bucket_name=f'quicksight-migration-{core.Aws.ACCOUNT_ID}',
block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
)
self.quicksight_migration_lambda_role = iam.Role(
self,
'quicksight-migration-lambda-role',
description='Role for the Quicksight dashboard migration Lambdas',
role_name='quicksight-migration-lambda-role',
max_session_duration=core.Duration.seconds(3600),
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'AllowAccess':
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'logs:CreateLogGroup', 'logs:CreateLogStream',
'logs:PutLogEvents'
],
resources=[
f'arn:aws:logs:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:*'
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["sts:AssumeRole", "iam:ListRoles"],
resources=[
"arn:aws:iam::*:role/quicksight-migration-*-assume-role"
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["s3:PutObject", "s3:ListBucket"],
resources=[
self.bucket.bucket_arn,
f"{self.bucket.bucket_arn}/*"
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["secrets:GetSecretValue"],
resources=[
f"arn:aws:secretsmanager:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:secret:*"
]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"quicksight:Create*", "quicksight:Delete*",
"quicksight:Describe*", "quicksight:List*",
"quicksight:Search*", "quicksight:Update*"
],
resources=["*"])
])
})
self.quicksight_migration_target_assume_role = iam.Role(
self,
'quicksight-migration-target-assume-role',
description=
'Role for the Quicksight dashboard migration Lambdas to assume',
role_name='quicksight-migration-target-assume-role',
max_session_duration=core.Duration.seconds(3600),
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'AllowAccess':
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"quicksight:Create*", "quicksight:Delete*",
"quicksight:Describe*", "quicksight:List*",
"quicksight:Search*", "quicksight:Update*"
],
resources=["*"]),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"ssm:GetParameter",
],
resources=["arn:aws:ssm:*:*:parameter/infra/config"])
])
})
self.quicksight_migration_target_assume_role.assume_role_policy.add_statements(
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=['sts:AssumeRole'],
principals=[iam.AccountPrincipal(core.Aws.ACCOUNT_ID)]))
# API Gateway to SQS
self.rest_api_role = iam.Role(
self,
"RestAPIRole",
assumed_by=iam.ServicePrincipal("apigateway.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSQSFullAccess")
])
self.queue = sqs.Queue(self,
"quicksight-migration-sqs-queue",
queue_name="quicksight-migration-sqs",
visibility_timeout=core.Duration.minutes(15))
self.integration_response = apigw.IntegrationResponse(
status_code="200",
response_templates={"application/json": ""},
response_parameters={
"method.response.header.Access-Control-Allow-Headers":
"'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'",
"method.response.header.Access-Control-Allow-Origin":
"'*'",
"method.response.header.Access-Control-Allow-Methods":
"'POST,OPTIONS'"
})
self.api_integration_options = apigw.IntegrationOptions(
credentials_role=self.rest_api_role,
integration_responses=[self.integration_response],
request_templates={
"application/json":
'Action=SendMessage&MessageBody=$util.urlEncode("$input.body")'
},
passthrough_behavior=apigw.PassthroughBehavior.NEVER,
request_parameters={
"integration.request.header.Content-Type":
"'application/x-www-form-urlencoded'"
})
self.api_resource_sqs_integration = apigw.AwsIntegration(
service="sqs",
integration_http_method="POST",
path="{}/{}".format(core.Aws.ACCOUNT_ID, self.queue.queue_name),
options=self.api_integration_options)
self.base_api = apigw.RestApi(
self,
'quicksight-migration-sqs',
rest_api_name='quicksight-migration-sqs',
default_cors_preflight_options=apigw.CorsOptions(
allow_origins=apigw.Cors.ALL_ORIGINS,
allow_methods=["POST", "OPTIONS"],
allow_headers=[
'Access-Control-Allow-Origin',
'Access-Control-Allow-Headers', 'Content-Type'
]))
self.base_api.root.add_method(
"POST",
self.api_resource_sqs_integration,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Headers':
True,
'method.response.header.Access-Control-Allow-Methods':
True,
'method.response.header.Access-Control-Allow-Origin': True
}
}])
self.quicksight_migration_lambda = _lambda.Function(
self,
'quicksight-migration-lambda',
handler='quicksight_migration.lambda_function.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
os.path.join(self.current_dir,
'../lambda/quicksight_migration/')),
function_name='quicksight_migration_lambda',
role=self.quicksight_migration_lambda_role,
timeout=core.Duration.minutes(15),
memory_size=1024,
environment={
'BUCKET_NAME': self.bucket.bucket_name,
'S3_KEY': 'None',
'INFRA_CONFIG_PARAM': '/infra/config',
'SQS_URL': self.queue.queue_url
})
self.sqs_event_source = event_sources.SqsEventSource(self.queue)
self.quicksight_migration_lambda.add_event_source(
self.sqs_event_source)
core.CfnOutput(self,
"MigrationAPIGatewayURL",
value=self.base_api.url,
description="Migration API GW URL")
def __init__(self, app: core.App, id: str) -> None:
super().__init__(app, id)
##################################
# Resource Property Config
##################################
# see: https://docs.aws.amazon.com/lambda/latest/dg/
# with-sqs.html#events-sqs-queueconfig
domain_scan_timeout = 900
queue_visibility_timeout = 6 * domain_scan_timeout
# tldextract needs to cache tld list after request and /tmp is writable in Lamdba
tld_cache = os.path.join('/tmp', '.tld_set')
##################################
# Domain Gatherer Lambda and Queue
##################################
# create queue
domain_queue = sqs.Queue(
self,
'domain-queue',
visibility_timeout=core.Duration.seconds(queue_visibility_timeout),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=5,
queue=sqs.Queue(self,
'domain-queue-dlq',
retention_period=core.Duration.days(5))))
# create lambda to gather domains
domain_gatherer_lambda = lambda_.Function(
self,
"domain-gatherer",
code=lambda_.Code.from_asset(
'lambda-releases/domain-gatherer.zip'),
handler="handler.main",
timeout=core.Duration.seconds(600),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=400)
# set env vars
domain_gatherer_lambda.add_environment('SQS_URL',
domain_queue.queue_url)
domain_gatherer_lambda.add_environment('TLDEXTRACT_CACHE', tld_cache)
# provide lambda with execution role
domain_gatherer_lambda_exec_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'lambda:InvokeFunction', 'sqs:SendMessage',
'sqs:DeleteMessage', 'sqs:SendMessageBatch',
'sqs:SetQueueAttributes', "sqs:GetQueueAttributes",
"sqs:GetQueueUrl", 'sqs:GetQueueAttributes'
],
resources=[domain_queue.queue_arn])
domain_gatherer_lambda.add_to_role_policy(
domain_gatherer_lambda_exec_policy)
# allow lambda to send messages to queue
domain_queue.grant_send_messages(domain_gatherer_lambda)
# create rule to run the lambda every Friday
rule = events.Rule(
self,
"Rule",
schedule=events.Schedule.cron(minute='0',
hour='18',
month='*',
week_day='FRI',
year='*'),
)
rule.add_target(targets.LambdaFunction(domain_gatherer_lambda))
##################################
# Domain Scan Lambda and Results Bucket
##################################
# create lambda to scan domains
domain_scanner_lambda = lambda_.Function(
self,
"domain-scanner",
code=lambda_.Code.from_asset('lambda-releases/domain-scanner.zip'),
handler="handler.main",
timeout=core.Duration.seconds(domain_scan_timeout),
runtime=lambda_.Runtime.PYTHON_3_7,
memory_size=1000)
# create sqs event source for domain scan lambda
domain_scanner_lambda.add_event_source(
sources.SqsEventSource(domain_queue, batch_size=2))
# create s3 bucket to put results
bucket = s3.Bucket(self,
'results-bucket',
versioned=False,
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess(
block_public_acls=False,
ignore_public_acls=False,
block_public_policy=True,
restrict_public_buckets=True))
# grant s3:PUT to the pa11y lambda
bucket.grant_put(domain_scanner_lambda)
# set an env var for bucket name
domain_scanner_lambda.add_environment('BUCKET_NAME',
bucket.bucket_name)
# create execution role for domain scanner lambda
domain_scanner_lambda_exec_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
'lambda:InvokeFunction', 'sqs:SendMessage',
'sqs:DeleteMessage', 'sqs:SendMessageBatch',
'sqs:SetQueueAttributes', "sqs:GetQueueAttributes",
"sqs:GetQueueUrl", 'sqs:GetQueueAttributes', "s3:PutObject",
"s3:PutObjectAcl"
],
resources=[bucket.bucket_arn])
domain_scanner_lambda.add_to_role_policy(
domain_scanner_lambda_exec_policy)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Message timeout; used by SQS and Lambda
message_timeout = core.Duration.seconds(15)
# SQS queue that the Raspberry Pi will write to
queue = sqs.Queue(
self,
'Queue',
visibility_timeout=message_timeout,
receive_message_wait_time=core.Duration.seconds(20),
retention_period=core.Duration.hours(1),
)
# DynamoDB table that the web app will read from
icao_address = dynamodb.Attribute(
name='IcaoAddress',
type=dynamodb.AttributeType.STRING,
)
table = dynamodb.Table(
self,
'Table',
partition_key=icao_address,
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY,
)
database = timestream.CfnDatabase(
self,
'Database',
database_name='aircraft-database',
)
table2 = timestream.CfnTable(self,
'Table2',
database_name=database.ref,
table_name='aircraft-table',
retention_properties={
'MemoryStoreRetentionPeriodInHours':
1,
'MagneticStoreRetentionPeriodInDays':
1,
})
# IAM user for the Raspberry Pi
user = iam.User(self, 'RaspberryPi')
queue.grant_send_messages(user)
access_key = iam.CfnAccessKey(
self,
'AccessKey',
user_name=user.user_name,
)
# IAM role for Lambda function, so it can write to DynamoDB
lambda_role = iam.Role(
self,
'LambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
],
)
lambda_role.add_to_policy(
iam.PolicyStatement(
actions=[
'timestream:CancelQuery', 'timestream:DescribeEndpoints',
'timestream:DescribeTable', 'timestream:ListMeasures',
'timestream:Select', 'timestream:WriteRecords'
],
resources=['*'], # TODO: narrow down permissions
))
table.grant_read_write_data(lambda_role)
# Integration between SQS and Lambda
event = lambda_event_sources.SqsEventSource(
queue=queue,
batch_size=10,
)
# Lambda function that processes messages from SQS queue and updates DynamoDB table
import_function = lambda_.Function(
self,
'ImportFunction',
description='Reads SQS messages and writes to DynamoDB',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.from_asset('lambda_import/'),
timeout=message_timeout,
handler='index.handler',
role=lambda_role,
events=[event],
environment={
'TABLE_NAME': table2.ref,
},
)
# TODO: add custom log group
# TODO: add metric filters for number of succesfull updates and failed updates
# Lambda function that reads from DynamoDB and returns data to API Gateway
api_function = lambda_.Function(
self,
'ApiFunction',
description='Reads from DynamoDB and returns to API GW',
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.from_asset('lambda_api/'),
timeout=message_timeout,
handler='index.handler',
role=lambda_role,
environment={
'TABLE_NAME': table.table_name,
},
)
# API Gateway for requesting aircraft data
api = apigateway.RestApi(
self,
'Api',
endpoint_types=[apigateway.EndpointType.REGIONAL],
cloud_watch_role=False,
)
aircraft_resource = api.root.add_resource('aircraft')
aircraft_resource.add_method(
http_method='GET',
integration=apigateway.LambdaIntegration(
api_function,
proxy=True,
),
)
# Static website
bucket = s3.Bucket(self, 'StaticWebsite')
s3_deployment.BucketDeployment(
self,
'Deployment',
sources=[
s3_deployment.Source.asset('html/'),
],
destination_bucket=bucket,
)
# Permissions between CloudFront and S3
origin_identity = cloudfront.OriginAccessIdentity(self, 'Identity')
bucket.grant_read(origin_identity.grant_principal)
# CloudFront distribution pointing to both S3 and API Gateway
s3_origin = cloudfront.SourceConfiguration(
s3_origin_source=cloudfront.S3OriginConfig(
s3_bucket_source=bucket,
origin_access_identity=origin_identity,
),
behaviors=[
cloudfront.Behavior(
default_ttl=core.Duration.days(0),
min_ttl=core.Duration.days(0),
max_ttl=core.Duration.days(31),
is_default_behavior=True,
)
])
api_origin = cloudfront.SourceConfiguration(
origin_path='/{}'.format(api.deployment_stage.stage_name),
custom_origin_source=cloudfront.CustomOriginConfig(
domain_name='{}.execute-api.{}.{}'.format(
api.rest_api_id, self.region, self.url_suffix), ),
behaviors=[
cloudfront.Behavior(
default_ttl=core.Duration.seconds(0),
min_ttl=core.Duration.seconds(0),
max_ttl=core.Duration.seconds(0),
path_pattern='/aircraft/*',
)
])
domain_name = self.node.try_get_context('domain_name')
# If domain name is specified, create a certificate and alias configuration for CloudFront
if domain_name is None:
alias_configuration = None
else:
subdomain = 'aircraft.{}'.format(domain_name)
zone = route53.HostedZone.from_lookup(
self,
'Zone',
domain_name=domain_name,
)
certificate = acm.DnsValidatedCertificate(
self,
'Certificate',
domain_name=subdomain,
hosted_zone=zone,
region='us-east-1',
)
alias_configuration = cloudfront.AliasConfiguration(
acm_cert_ref=certificate.certificate_arn,
names=[subdomain],
)
distribution = cloudfront.CloudFrontWebDistribution(
self,
'CDN',
price_class=cloudfront.PriceClass.PRICE_CLASS_ALL,
alias_configuration=alias_configuration,
origin_configs=[
s3_origin,
api_origin,
],
)
# If domain name is specified, create a DNS record for CloudFront
if domain_name is not None:
route53.ARecord(
self,
'DnsRecord',
record_name=subdomain,
target=route53.AddressRecordTarget.from_alias(
alias_target=route53_targets.CloudFrontTarget(
distribution)),
zone=zone,
)
# Outputs that are needed on the Raspberry Pi
core.CfnOutput(
self,
'QueueUrl',
value=queue.queue_url,
)
core.CfnOutput(
self,
'AccessKeyId',
value=access_key.ref,
)
core.CfnOutput(
self,
'SecretAccessKey',
value=access_key.attr_secret_access_key,
)
core.CfnOutput(
self,
'Region',
value=self.region,
)
def __init__(self, scope: core.Construct, id: str, instance_id: str,
contact_flow_id: str, source_phone_number: str, timeout: int,
**kwargs):
super().__init__(scope, id, **kwargs)
web_bucket = _s3.Bucket(self,
"StaticWebBucket",
website_index_document="index.html",
website_error_document="index.html",
removal_policy=core.RemovalPolicy.DESTROY,
public_read_access=True)
core.CfnOutput(self,
'WebBucketUrl',
value=web_bucket.bucket_domain_name)
web_distribution = _clf.CloudFrontWebDistribution(
self,
'StaticWebDistribution',
origin_configs=[
_clf.SourceConfiguration(
s3_origin_source=_clf.S3OriginConfig(
s3_bucket_source=web_bucket),
behaviors=[_clf.Behavior(is_default_behavior=True)])
],
viewer_protocol_policy=_clf.ViewerProtocolPolicy.REDIRECT_TO_HTTPS)
_s3d.BucketDeployment(
self,
"S3StaticWebContentDeploymentWithInvalidation",
sources=[
_s3d.Source.asset(
f"{pathlib.Path(__file__).parent.absolute()}/site-content/build"
)
],
destination_bucket=web_bucket,
distribution=web_distribution,
distribution_paths=["/*"])
file_bucket = _s3.Bucket(self,
"FileBucket",
removal_policy=core.RemovalPolicy.DESTROY)
call_dead_letter_queue = _sqs.Queue(self,
"CallDeadLetterQueue",
fifo=True,
content_based_deduplication=True)
call_sqs_queue = _sqs.Queue(
self,
"CallSqsQueue",
fifo=True,
content_based_deduplication=True,
visibility_timeout=core.Duration.seconds(120),
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=1, queue=call_dead_letter_queue))
async_call_dead_letter_queue = _sqs.Queue(
self,
"AsyncCallDeadLetterQueue",
fifo=True,
content_based_deduplication=True)
async_callout_queue = _sqs.Queue(
self,
"AsyncCalloutQueue",
fifo=True,
content_based_deduplication=True,
visibility_timeout=core.Duration.seconds(120),
dead_letter_queue=_sqs.DeadLetterQueue(
max_receive_count=1, queue=async_call_dead_letter_queue))
call_job_complete_sns_topic = _sns.Topic(
self, "CallJobCompleteSnsTopic", display_name="CallJobCompletion")
call_result_table = _dynamodb.Table(
self,
"CallResultDynamodbTable",
billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST,
partition_key=_dynamodb.Attribute(
name="task_id", type=_dynamodb.AttributeType.STRING),
sort_key=_dynamodb.Attribute(name="receiver_id",
type=_dynamodb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.DESTROY)
callout_record_table = _dynamodb.Table(
self,
"CallTaskDynamodbTable",
billing_mode=_dynamodb.BillingMode.PAY_PER_REQUEST,
partition_key=_dynamodb.Attribute(
name="task_id", type=_dynamodb.AttributeType.STRING),
sort_key=_dynamodb.Attribute(name="created_at",
type=_dynamodb.AttributeType.NUMBER),
removal_policy=core.RemovalPolicy.DESTROY)
callout_record_table.add_global_secondary_index(
partition_key=_dynamodb.Attribute(
name='call_type', type=_dynamodb.AttributeType.STRING),
sort_key=_dynamodb.Attribute(name='created_at',
type=_dynamodb.AttributeType.NUMBER),
index_name='CallTypeCreatedAtGlobalIndex',
projection_type=_dynamodb.ProjectionType.ALL)
python_function_layer = _lambda.LayerVersion(
self,
"LambdaPythonFunctionLayer",
code=_lambda.Code.asset("aws_callouts_cdk/layer/_python"),
compatible_runtimes=[
_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8
],
license="Available under the MIT-0 license")
nodejs_function_layer = _lambda.LayerVersion(
self,
"LambdaNodeJsFunctionLayer",
code=_lambda.Code.asset("aws_callouts_cdk/layer/_nodejs"),
compatible_runtimes=[
_lambda.Runtime.NODEJS_10_X, _lambda.Runtime.NODEJS_12_X
],
license="Available under the MIT-0 license")
global_python_function_arguments = {
"code": _lambda.Code.asset("aws_callouts_cdk/src/python"),
"layers": [python_function_layer],
"runtime": _lambda.Runtime.PYTHON_3_7
}
global_nodeje_function_arguments = {
"code": _lambda.Code.asset("aws_callouts_cdk/src/nodejs"),
"layers": [nodejs_function_layer],
"runtime": _lambda.Runtime.NODEJS_12_X
}
get_callout_job_function = _lambda.Function(
self,
"GetCalloutJobFunction",
handler="get_call_job.lambda_handler",
**global_python_function_arguments)
get_callout_job_function.add_environment(key="S3Bucket",
value=file_bucket.bucket_name)
file_bucket.grant_read(get_callout_job_function)
callout_function = _lambda.Function(self,
"CalloutFunction",
handler="send_call.lambda_handler",
**global_python_function_arguments)
callout_function.add_environment(
key="ContactFlowArn",
value=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/contact-flow/{contact_flow_id}"
)
callout_function.add_environment(key="SourcePhoneNumber",
value=source_phone_number)
callout_function.add_environment(key="ExcelFileBucket",
value=file_bucket.bucket_name)
callout_function.add_environment(key="AsynCalloutQueueUrl",
value=async_callout_queue.queue_url)
callout_function.add_to_role_policy(statement=_iam.PolicyStatement(
resources=[
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}/*"
],
actions=["connect:StartOutboundVoiceContact"]))
callout_function.add_event_source(source=_les.SqsEventSource(
queue=async_callout_queue, batch_size=1))
'arn:aws:connect:751225572132:ap-southeast-2:instance/9d0c7cc5-7d2a-42e4-a3dd-70f402e0d040'
file_bucket.grant_read_write(callout_function)
response_handler_function = _lambda.Function(
self,
"ResponseHandlerFunction",
handler="response_handler.lambda_handler",
**global_python_function_arguments)
response_handler_function.add_permission(
id="ResponseHandlerFunctionLambdaInvokePermission",
principal=_iam.ServicePrincipal(service="connect.amazonaws.com"),
action="lambda:InvokeFunction",
source_account=self.account,
source_arn=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}"
)
send_task_success_function = _lambda.Function(
self,
"SendTaskSuccessFunction",
handler="send_task_success.lambda_handler",
**global_python_function_arguments)
send_task_success_function.add_permission(
id="SendTaskSuccessFunctionLambdaInvokePermission",
principal=_iam.ServicePrincipal(service="connect.amazonaws.com"),
action="lambda:InvokeFunction",
source_account=self.account,
source_arn=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}"
)
get_call_result_function = _lambda.Function(
self,
"GetCallResultFunction",
handler="get_call_result.lambda_handler",
memory_size=512,
**global_python_function_arguments)
get_call_result_function.add_environment(
key="CallResultDynamoDBTable", value=call_result_table.table_name)
get_call_result_function.add_environment(key="S3Bucket",
value=file_bucket.bucket_name)
call_result_table.grant_read_data(grantee=get_call_result_function)
file_bucket.grant_read_write(get_call_result_function)
iterator_function = _lambda.Function(
self,
"IteratorFunction",
handler="iterator.lambda_handler",
**global_python_function_arguments)
iterator_function.add_permission(
id="IteratorFunctionLambdaInvokePermission",
principal=_iam.ServicePrincipal(service="connect.amazonaws.com"),
action="lambda:InvokeFunction",
source_account=self.account,
source_arn=
f"arn:aws:connect:{self.region}:{self.account}:instance/{instance_id}"
)
create_appsync_call_task_function = _lambda.Function(
self,
"CreateAppSyncCallTaskFunction",
handler="create_appsync_call_task.lambda_handler",
**global_nodeje_function_arguments)
create_appsync_call_task_function.add_environment(
key="CallSqsQueueUrl", value=call_sqs_queue.queue_url)
create_appsync_call_task_function.add_environment(
key="CallRecordTableName", value=callout_record_table.table_name)
call_sqs_queue.grant_send_messages(create_appsync_call_task_function)
callout_record_table.grant_write_data(
create_appsync_call_task_function)
create_call_report_record_function = _lambda.Function(
self,
"CreateCallReportRecordFunction",
handler="create_call_report_record.lambda_handler",
**global_nodeje_function_arguments)
create_excel_call_task_function = _lambda.Function(
self,
"CreateExcelCallTaskFunction",
handler="create_excel_call_task.lambda_handler",
**global_python_function_arguments)
create_excel_call_task_function.add_environment(
key="CallSqsQueueUrl", value=call_sqs_queue.queue_url)
call_sqs_queue.grant_send_messages(create_excel_call_task_function)
create_excel_call_task_function.add_event_source(
source=_les.S3EventSource(bucket=file_bucket,
events=[_s3.EventType.OBJECT_CREATED],
filters=[
_s3.NotificationKeyFilter(
prefix="call_task",
suffix=".xlsx")
]))
start_callout_flow_function = _lambda.Function(
self,
"StartCalloutFlowFunction",
handler="start_call_out_flow.lambda_handler",
reserved_concurrent_executions=1,
**global_python_function_arguments)
start_callout_flow_function.add_environment(
key="CallSqsQueueUrl", value=call_sqs_queue.queue_url)
start_callout_flow_function.add_environment(
key="ResponseHandlerFunctionArn",
value=response_handler_function.function_arn)
start_callout_flow_function.add_environment(
key="IteratorFunctionArn", value=iterator_function.function_arn)
start_callout_flow_function.add_environment(
key="SendTaskSuccessFunctionArn",
value=send_task_success_function.function_arn)
start_callout_flow_function.add_environment(
key="S3Bucket", value=file_bucket.bucket_name)
start_callout_flow_function.add_event_source(
source=_les.SqsEventSource(queue=call_sqs_queue, batch_size=1))
file_bucket.grant_read_write(start_callout_flow_function)
call_state_machine_definition = {
"Comment":
"Reading messages from an SQS queue and iteratively processing each message.",
"StartAt": "Start",
"States": {
"Start": {
"Type": "Pass",
"Next": "Process Call Messages"
},
"Process Call Messages": {
"Type": "Map",
"Next": "Get Call Result",
"InputPath": "$",
"ItemsPath": "$",
"OutputPath": "$.[0]",
"Iterator": {
"StartAt": "Get Call out job",
"States": {
"Get Call out job": {
"Type": "Task",
"Resource":
get_callout_job_function.function_arn,
"Next": "Callout with AWS Connect"
},
"Callout with AWS Connect": {
"Type":
"Task",
"Resource":
"arn:aws:states:::sqs:sendMessage.waitForTaskToken",
"TimeoutSeconds":
timeout,
"Parameters": {
"QueueUrl": async_callout_queue.queue_url,
"MessageGroupId": "1",
"MessageBody": {
"Message.$": "$",
"TaskToken.$": "$$.Task.Token"
}
},
"Catch": [{
"ErrorEquals": ["States.Timeout"],
"ResultPath": None,
"Next": "Call Timeout"
}],
"Next":
"Save call result"
},
"Call Timeout": {
"Type": "Pass",
"ResultPath": None,
"Next": "Save call result"
},
"Save call result": {
"Type": "Task",
"Resource":
"arn:aws:states:::dynamodb:putItem",
"Parameters": {
"TableName": call_result_table.table_name,
"Item": {
"receiver_id": {
"S.$": "$.receiver_id"
},
"task_id": {
"S.$": "$.task_id"
},
"username": {
"S.$": "$.username"
},
"phone_number": {
"S.$": "$.phone_number"
},
"status": {
"S.$": "$.status"
},
"answers": {
"S.$": "$.answers"
},
"error": {
"S.$": "$.error"
},
"call_at": {
"S.$": "$.call_at"
}
}
},
"ResultPath": "$.Result",
"OutputPath": "$.task_id",
"End": True
}
}
}
},
"Get Call Result": {
"Type": "Task",
"Resource": get_call_result_function.function_arn,
"Next": "Create Call Report Record"
},
"Create Call Report Record": {
"Type": "Task",
"Resource":
create_call_report_record_function.function_arn,
"Next": "Send Completion message to SNS"
},
"Send Completion message to SNS": {
"Type": "Task",
"Resource": "arn:aws:states:::sns:publish",
"Parameters": {
"TopicArn": call_job_complete_sns_topic.topic_arn,
"Message.$": "$"
},
"Next": "Finish"
},
"Finish": {
"Type": "Succeed"
}
}
}
callout_state_machine_role = _iam.Role(
self,
"CalloutStatesExecutionRole",
assumed_by=_iam.ServicePrincipal(
f"states.{self.region}.amazonaws.com"))
callout_state_machine_role.add_to_policy(
_iam.PolicyStatement(
actions=[
"sqs:SendMessage", "dynamodb:PutItem",
"lambda:InvokeFunction", "SNS:Publish"
],
resources=[
async_callout_queue.queue_arn, call_result_table.table_arn,
get_callout_job_function.function_arn,
get_call_result_function.function_arn,
call_job_complete_sns_topic.topic_arn,
create_appsync_call_task_function.function_arn,
create_call_report_record_function.function_arn
]))
callout_state_machine = _sfn.CfnStateMachine(
self,
"CalloutStateMachine",
role_arn=callout_state_machine_role.role_arn,
definition_string=json.dumps(call_state_machine_definition))
send_task_success_function.add_to_role_policy(
_iam.PolicyStatement(actions=["states:SendTaskSuccess"],
resources=[callout_state_machine.ref]))
start_callout_flow_function.add_environment(
key="CalloutStateMachineArn", value=callout_state_machine.ref)
start_callout_flow_function.add_to_role_policy(
_iam.PolicyStatement(effect=_iam.Effect.ALLOW,
resources=[callout_state_machine.ref],
actions=['states:StartExecution']))
user_pool = _cognito.UserPool(
self, "UserPool", sign_in_type=_cognito.SignInType.USERNAME)
user_pool_client = _cognito.UserPoolClient(self,
"UserPoolClient",
user_pool=user_pool)
appsync_api = _appsync.GraphQLApi(
self,
"AppSyncApi",
name="AWSCalloutApi",
user_pool_config=_appsync.UserPoolConfig(
user_pool=user_pool,
default_action=_appsync.UserPoolDefaultAction.ALLOW),
log_config=_appsync.LogConfig(
field_log_level=_appsync.FieldLogLevel.ALL),
schema_definition_file=
f"{pathlib.Path(__file__).parent.absolute()}/schema.graphql")
callout_record_ddb_ds = appsync_api.add_dynamo_db_data_source(
name="CalloutRecordDdb",
description="Callout Record DynamoDB Data Source",
table=callout_record_table)
callout_record_ddb_ds.create_resolver(
type_name="Query",
field_name="getLatestCallTaskRecords",
request_mapping_template=_appsync.MappingTemplate.from_string(
'{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"TASK"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}'
),
response_mapping_template=_appsync.MappingTemplate.
dynamo_db_result_list())
callout_record_ddb_ds.create_resolver(
type_name="Query",
field_name="getLatestCallReportRecords",
request_mapping_template=_appsync.MappingTemplate.from_string(
'{"version":"2017-02-28","operation":"Query","index":"CallTypeCreatedAtGlobalIndex","query":{"expression":"call_type = :call_type","expressionValues":{":call_type":{"S":"REPORT"}}},"scanIndexForward":false,"limit":${ctx.args.limit}}'
),
response_mapping_template=_appsync.MappingTemplate.
dynamo_db_result_list())
callout_record_ddb_ds.create_resolver(
type_name="Mutation",
field_name="createCallReport",
request_mapping_template=_appsync.MappingTemplate.from_string(
'{"version":"2017-02-28","operation":"PutItem","key":{"task_id":{"S":"${ctx.args.report.task_id}"},"created_at":{"N":"${ctx.args.report.created_at}"}},"attributeValues":$util.dynamodb.toMapValuesJson($ctx.args.report)}'
),
response_mapping_template=_appsync.MappingTemplate.
dynamo_db_result_item())
call_task_lambda_ds = appsync_api.add_lambda_data_source(
name="CallTaskLambda",
description="Call Task Lambda Data Source",
lambda_function=create_appsync_call_task_function)
call_task_lambda_ds.create_resolver(
type_name="Mutation",
field_name="createCallTask",
request_mapping_template=_appsync.MappingTemplate.lambda_request(
"$utils.toJson($ctx.args)"),
response_mapping_template=_appsync.MappingTemplate.lambda_result())
create_call_report_record_function.add_environment(
value=appsync_api.graph_ql_url, key="AppSyncGraphQlApiUrl")
create_call_report_record_function.add_to_role_policy(
statement=_iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=['appsync:GraphQL'],
resources=[
f"{appsync_api.arn}/types/Mutation/fields/createCallReport"
]))
core.CfnOutput(self,
id="OutputCallSqsQueue",
value=call_sqs_queue.queue_arn)
core.CfnOutput(self,
id="OutputCallJobCompletionSNSTopic",
value=call_job_complete_sns_topic.topic_arn)
core.CfnOutput(self,
id="OutputExcelFileS3Bucket",
value=file_bucket.bucket_name)
core.CfnOutput(self,
id="OutputStaticWebS3Bucket",
value=web_bucket.bucket_name)
core.CfnOutput(self,
id="OutputStaticWebUrl",
value=web_bucket.bucket_website_url)
identity_pool = _cognito.CfnIdentityPool(
self,
"IdentityPool",
allow_unauthenticated_identities=True,
cognito_identity_providers=[
_cognito.CfnIdentityPool.CognitoIdentityProviderProperty(
provider_name=user_pool.user_pool_provider_name,
client_id=user_pool_client.user_pool_client_id)
])
identity_pool_unauthorized_role = _iam.Role(
self,
'IdentityPoolUnAuthorizedRole',
assumed_by=_iam.FederatedPrincipal(
federated="cognito-identity.amazonaws.com",
assume_role_action="sts:AssumeRoleWithWebIdentity",
conditions={
"StringEquals": {
"cognito-identity.amazonaws.com:aud": identity_pool.ref
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "unauthenticated"
}
}))
identity_pool_unauthorized_role.add_to_policy(
_iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=["appsync:GraphQL"],
resources=[
f"{appsync_api.arn}/types/*",
# f"{appsync_api.arn}/types/Query/fields/getLatestCallTaskRecords",
# f"{appsync_api.arn}/types/Query/fields/getLatestCallReportRecords",
# f"{appsync_api.arn}/types/Mutation/fields/createCallRecord",
# f"{appsync_api.arn}/types/Subscription/fields/createCallTask",
# f"{appsync_api.arn}/types/Subscription/fields/createCallReport"
]))
_cognito.CfnIdentityPoolRoleAttachment(
self,
"CognitoIdentityPoolRoleAttachment",
identity_pool_id=identity_pool.ref,
roles={
"unauthenticated": identity_pool_unauthorized_role.role_arn
})
core.CfnOutput(self, id="UserPoolId", value=user_pool.user_pool_id)
core.CfnOutput(self,
id="UserPoolClientId",
value=user_pool_client.user_pool_client_id)
core.CfnOutput(self, id="IdentityPoolId", value=identity_pool.ref)
