def _create_lambda_fn(self, envs: EnvSettings, memory_size: int,
queue: Queue):
is_app_only = self.node.try_get_context("is_app_only")
if is_app_only == "true":
code = Code.from_asset(
path="../backend/functions/worker/.serverless/main.zip")
else:
code = Code.from_cfn_parameters()
function = Function(
self,
f"data-processing-worker-{memory_size}",
function_name=f"{envs.project_name}-data-processing-{memory_size}",
code=code,
runtime=Runtime.PYTHON_3_8,
handler="handler.main",
environment={
"AWS_STORAGE_BUCKET_NAME":
self.app_bucket.bucket_name,
"IMAGE_SCRAPING_FETCH_TIMEOUT":
"15",
"AWS_IMAGE_STORAGE_BUCKET_NAME":
self.resize_lambda_image_bucket.bucket_name,
"AWS_IMAGE_STATIC_URL":
self.resize_lambda_image_bucket.bucket_website_url,
"BACKEND_URL":
self.backend_url,
"LAMBDA_AUTH_TOKEN":
self.lambda_auth_token.secret_value.to_string(),
},
memory_size=memory_size,
timeout=Duration.seconds(300),
tracing=Tracing.ACTIVE,
)
function.add_event_source(SqsEventSource(queue, batch_size=1))
self.app_bucket.grant_read_write(function.role)
self.resize_lambda_image_bucket.grant_read_write(function.role)
return function, code
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
with open("stack/config.yml", 'r') as stream:
configs = yaml.safe_load(stream)
### S3 core
images_S3_bucket = _s3.Bucket(self, "ICS_IMAGES")
images_S3_bucket.add_cors_rule(
allowed_methods=[_s3.HttpMethods.POST],
allowed_origins=["*"] # add API gateway web resource URL
)
### SQS core
image_deadletter_queue = _sqs.Queue(self, "ICS_IMAGES_DEADLETTER_QUEUE")
image_queue = _sqs.Queue(self, "ICS_IMAGES_QUEUE",
dead_letter_queue={
"max_receive_count": configs["DeadLetterQueue"]["MaxReceiveCount"],
"queue": image_deadletter_queue
})
### api gateway core
api_gateway = RestApi(self, 'ICS_API_GATEWAY', rest_api_name='ImageContentSearchApiGateway')
api_gateway_resource = api_gateway.root.add_resource(configs["ProjectName"])
api_gateway_landing_page_resource = api_gateway_resource.add_resource('web')
api_gateway_get_signedurl_resource = api_gateway_resource.add_resource('signedUrl')
api_gateway_image_search_resource = api_gateway_resource.add_resource('search')
### landing page function
get_landing_page_function = Function(self, "ICS_GET_LANDING_PAGE",
function_name="ICS_GET_LANDING_PAGE",
runtime=Runtime.PYTHON_3_7,
handler="main.handler",
code=Code.asset("./src/landingPage"))
get_landing_page_integration = LambdaIntegration(
get_landing_page_function,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}])
api_gateway_landing_page_resource.add_method('GET', get_landing_page_integration,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}])
### cognito
required_attribute = _cognito.StandardAttribute(required=True)
users_pool = _cognito.UserPool(self, "ICS_USERS_POOL",
auto_verify=_cognito.AutoVerifiedAttrs(email=True), #required for self sign-up
standard_attributes=_cognito.StandardAttributes(email=required_attribute), #required for self sign-up
self_sign_up_enabled=configs["Cognito"]["SelfSignUp"])
user_pool_app_client = _cognito.CfnUserPoolClient(self, "ICS_USERS_POOL_APP_CLIENT",
supported_identity_providers=["COGNITO"],
allowed_o_auth_flows=["implicit"],
allowed_o_auth_scopes=configs["Cognito"]["AllowedOAuthScopes"],
user_pool_id=users_pool.user_pool_id,
callback_ur_ls=[api_gateway_landing_page_resource.url],
allowed_o_auth_flows_user_pool_client=True,
explicit_auth_flows=["ALLOW_REFRESH_TOKEN_AUTH"])
user_pool_domain = _cognito.UserPoolDomain(self, "ICS_USERS_POOL_DOMAIN",
user_pool=users_pool,
cognito_domain=_cognito.CognitoDomainOptions(domain_prefix=configs["Cognito"]["DomainPrefix"]))
### get signed URL function
get_signedurl_function = Function(self, "ICS_GET_SIGNED_URL",
function_name="ICS_GET_SIGNED_URL",
environment={
"ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name,
"DEFAULT_SIGNEDURL_EXPIRY_SECONDS": configs["Functions"]["DefaultSignedUrlExpirySeconds"]
},
runtime=Runtime.PYTHON_3_7,
handler="main.handler",
code=Code.asset("./src/getSignedUrl"))
get_signedurl_integration = LambdaIntegration(
get_signedurl_function,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}])
api_gateway_get_signedurl_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER",
rest_api_id=api_gateway_get_signedurl_resource.rest_api.rest_api_id,
name="ICS_API_GATEWAY_GET_SIGNED_URL_AUTHORIZER",
type="COGNITO_USER_POOLS",
identity_source="method.request.header.Authorization",
provider_arns=[users_pool.user_pool_arn])
api_gateway_get_signedurl_resource.add_method('GET', get_signedurl_integration,
authorization_type=AuthorizationType.COGNITO,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}]
).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_get_signedurl_authorizer.ref)
images_S3_bucket.grant_put(get_signedurl_function, objects_key_pattern="new/*")
### image massage function
image_massage_function = Function(self, "ICS_IMAGE_MASSAGE",
function_name="ICS_IMAGE_MASSAGE",
timeout=core.Duration.seconds(6),
runtime=Runtime.PYTHON_3_7,
environment={"ICS_IMAGE_MASSAGE": image_queue.queue_name},
handler="main.handler",
code=Code.asset("./src/imageMassage"))
images_S3_bucket.grant_write(image_massage_function, "processed/*")
images_S3_bucket.grant_delete(image_massage_function, "new/*")
images_S3_bucket.grant_read(image_massage_function, "new/*")
new_image_added_notification = _s3notification.LambdaDestination(image_massage_function)
images_S3_bucket.add_event_notification(_s3.EventType.OBJECT_CREATED,
new_image_added_notification,
_s3.NotificationKeyFilter(prefix="new/")
)
image_queue.grant_send_messages(image_massage_function)
### image analyzer function
image_analyzer_function = Function(self, "ICS_IMAGE_ANALYSIS",
function_name="ICS_IMAGE_ANALYSIS",
runtime=Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(10),
environment={
"ICS_IMAGES_BUCKET": images_S3_bucket.bucket_name,
"DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"],
"REGION": core.Aws.REGION,
},
handler="main.handler",
code=Code.asset("./src/imageAnalysis"))
image_analyzer_function.add_event_source(_lambda_event_source.SqsEventSource(queue=image_queue, batch_size=10))
image_queue.grant_consume_messages(image_massage_function)
lambda_rekognition_access = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=["rekognition:DetectLabels", "rekognition:DetectModerationLabels"],
resources=["*"]
)
image_analyzer_function.add_to_role_policy(lambda_rekognition_access)
images_S3_bucket.grant_read(image_analyzer_function, "processed/*")
### API gateway finalizing
self.add_cors_options(api_gateway_get_signedurl_resource)
self.add_cors_options(api_gateway_landing_page_resource)
self.add_cors_options(api_gateway_image_search_resource)
### database
database_secret = _secrets_manager.Secret(self, "ICS_DATABASE_SECRET",
secret_name="rds-db-credentials/image-content-search-rds-secret",
generate_secret_string=_secrets_manager.SecretStringGenerator(
generate_string_key='password',
secret_string_template='{"username": "******"}',
exclude_punctuation=True,
exclude_characters='/@\" \\\'',
require_each_included_type=True
)
)
database = _rds.CfnDBCluster(self, "ICS_DATABASE",
engine=_rds.DatabaseClusterEngine.aurora_mysql(version=_rds.AuroraMysqlEngineVersion.VER_5_7_12).engine_type,
engine_mode="serverless",
database_name=configs["Database"]["Name"],
enable_http_endpoint=True,
deletion_protection=configs["Database"]["DeletionProtection"],
master_username=database_secret.secret_value_from_json("username").to_string(),
master_user_password=database_secret.secret_value_from_json("password").to_string(),
scaling_configuration=_rds.CfnDBCluster.ScalingConfigurationProperty(
auto_pause=configs["Database"]["Scaling"]["AutoPause"],
min_capacity=configs["Database"]["Scaling"]["Min"],
max_capacity=configs["Database"]["Scaling"]["Max"],
seconds_until_auto_pause=configs["Database"]["Scaling"]["SecondsToAutoPause"]
),
)
database_cluster_arn = "arn:aws:rds:{}:{}:cluster:{}".format(core.Aws.REGION, core.Aws.ACCOUNT_ID, database.ref)
secret_target = _secrets_manager.CfnSecretTargetAttachment(self,"ICS_DATABASE_SECRET_TARGET",
target_type="AWS::RDS::DBCluster",
target_id=database.ref,
secret_id=database_secret.secret_arn
)
secret_target.node.add_dependency(database)
### database function
image_data_function_role = _iam.Role(self, "ICS_IMAGE_DATA_FUNCTION_ROLE",
role_name="ICS_IMAGE_DATA_FUNCTION_ROLE",
assumed_by=_iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
_iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaVPCAccessExecutionRole"),
_iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"),
_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonRDSDataFullAccess")
]
)
image_data_function = Function(self, "ICS_IMAGE_DATA",
function_name="ICS_IMAGE_DATA",
runtime=Runtime.PYTHON_3_7,
timeout=core.Duration.seconds(5),
role=image_data_function_role,
environment={
"DEFAULT_MAX_CALL_ATTEMPTS": configs["Functions"]["DefaultMaxApiCallAttempts"],
"CLUSTER_ARN": database_cluster_arn,
"CREDENTIALS_ARN": database_secret.secret_arn,
"DB_NAME": database.database_name,
"REGION": core.Aws.REGION
},
handler="main.handler",
code=Code.asset("./src/imageData")
)
image_search_integration = LambdaIntegration(
image_data_function,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}])
api_gateway_image_search_authorizer = CfnAuthorizer(self, "ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER",
rest_api_id=api_gateway_image_search_resource.rest_api.rest_api_id,
name="ICS_API_GATEWAY_IMAGE_SEARCH_AUTHORIZER",
type="COGNITO_USER_POOLS",
identity_source="method.request.header.Authorization",
provider_arns=[users_pool.user_pool_arn])
api_gateway_image_search_resource.add_method('POST', image_search_integration,
authorization_type=AuthorizationType.COGNITO,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}]
).node.find_child('Resource').add_property_override('AuthorizerId', api_gateway_image_search_authorizer.ref)
lambda_access_search = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
actions=["translate:TranslateText"],
resources=["*"]
)
image_data_function.add_to_role_policy(lambda_access_search)
### custom resource
lambda_provider = Provider(self, 'ICS_IMAGE_DATA_PROVIDER',
on_event_handler=image_data_function
)
core.CustomResource(self, 'ICS_IMAGE_DATA_RESOURCE',
service_token=lambda_provider.service_token,
pascal_case_properties=False,
resource_type="Custom::SchemaCreation",
properties={
"source": "Cloudformation"
}
)
### event bridge
event_bus = _events.EventBus(self, "ICS_IMAGE_CONTENT_BUS")
event_rule = _events.Rule(self, "ICS_IMAGE_CONTENT_RULE",
rule_name="ICS_IMAGE_CONTENT_RULE",
description="The event from image analyzer to store the data",
event_bus=event_bus,
event_pattern=_events.EventPattern(resources=[image_analyzer_function.function_arn]),
)
event_rule.add_target(_event_targets.LambdaFunction(image_data_function))
event_bus.grant_put_events(image_analyzer_function)
image_analyzer_function.add_environment("EVENT_BUS", event_bus.event_bus_name)
### outputs
core.CfnOutput(self, 'CognitoHostedUILogin',
value='https://{}.auth.{}.amazoncognito.com/login?client_id={}&response_type=token&scope={}&redirect_uri={}'.format(user_pool_domain.domain_name, core.Aws.REGION, user_pool_app_client.ref, '+'.join(user_pool_app_client.allowed_o_auth_scopes), api_gateway_landing_page_resource.url),
description='The Cognito Hosted UI Login Page'
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
"""
Create Lambda Layer
The packages should be stored in `python/lib/python3.7/site-packages`
which translates to `/opt/python/lib/python3.7/site-packages` in AWS Lambda
Refer here: https://stackoverflow.com/a/58702328/7999204
"""
python_deps_layer = LayerVersion(
self,
"PythonDepsLayer",
code=Code.from_asset("./python-deps-layer"),
compatible_runtimes=[PYTHON_RUNTIME],
description="A layer that contains Python Dependencies",
)
"""
Create DynamoDB Tables
"""
poll_table = Table(
self,
"PollTable",
partition_key=Attribute(name="id", type=AttributeType.STRING),
sort_key=Attribute(name="SK", type=AttributeType.STRING),
read_capacity=10,
write_capacity=10,
stream=StreamViewType.NEW_IMAGE,
)
# DynamoDB Lambda consumer worker
aggregate_votes_function = Function(
self,
"AggregateVotesLambda",
handler="ddb_stream.aggregate_vote_table",
runtime=PYTHON_RUNTIME,
code=Code.asset("./backend"),
layers=[python_deps_layer],
timeout=core.Duration.seconds(30),
)
aggregate_votes_function.add_environment("POLL_TABLE", poll_table.table_name)
# DynamoDB Stream (Lambda Event Source)
poll_table.grant_stream_read(aggregate_votes_function)
poll_table.grant_read_write_data(aggregate_votes_function)
ddb_aggregate_votes_event_source = DynamoEventSource(
poll_table, starting_position=StartingPosition.LATEST
)
aggregate_votes_function.add_event_source(ddb_aggregate_votes_event_source)
# DynamoDB main_page GSI
poll_table.add_global_secondary_index(
partition_key=Attribute(name="PK2", type=AttributeType.STRING),
projection_type=ProjectionType.INCLUDE,
index_name=MAIN_PAGE_GSI,
non_key_attributes=["date", "question", "result"],
)
"""
Create AWS Cognito User Pool
"""
self.users = UserPool(self, "vote-user")
"""
HTTP API API Gateway with CORS
"""
api = HttpApi(
self,
"VoteHttpApi",
cors_preflight={
"allow_headers": ["*"],
"allow_methods": [
HttpMethod.GET,
HttpMethod.HEAD,
HttpMethod.OPTIONS,
HttpMethod.POST,
],
"allow_origins": ["*"],
"max_age": core.Duration.days(10),
},
)
"""
HTTP API Lambda functions
"""
get_all_votes_function = api_lambda_function(
self,
"GetAllVoteLambda",
"api.get_all_votes",
api,
"/vote",
GET,
[python_deps_layer],
[poll_table],
)
poll_table.grant_read_data(get_all_votes_function)
get_vote_function = api_lambda_function(
self,
"GetVoteLambda",
"api.get_vote_by_id",
api,
"/vote/{vote_id}",
GET,
[python_deps_layer],
[poll_table],
)
poll_table.grant_read_data(get_vote_function)
create_poll_function = api_lambda_function(
self,
"CreatePollLambda",
"api.create_poll",
api,
"/vote",
POST,
[python_deps_layer],
[poll_table],
)
poll_table.grant_write_data(create_poll_function)
post_vote_function = api_lambda_function(
self,
"PostVoteLambda",
"api.vote",
api,
"/vote/{vote_id}",
POST,
[python_deps_layer],
[poll_table],
)
"""
Create SQS Queues
"""
voting_queue = Queue(self, "voting-queue")
# SQS Consumer worker
voting_to_ddb_function = Function(
self,
"VotingToDDBLambda",
handler="sqs_worker.insert_to_vote_db_table",
runtime=PYTHON_RUNTIME,
code=Code.asset("./backend"),
layers=[python_deps_layer],
)
voting_to_ddb_function.add_environment("POLL_TABLE", poll_table.table_name)
# SQS Queue to Lambda trigger mapping
voting_to_ddb_event_source = SqsEventSource(voting_queue)
voting_to_ddb_function.add_event_source(voting_to_ddb_event_source)
poll_table.grant_read_write_data(voting_to_ddb_function)
voting_queue.grant_send_messages(post_vote_function)
post_vote_function.add_environment("VOTING_QUEUE_URL", voting_queue.queue_url)
core.CfnOutput(self, "api-domain", value=api.url)