def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Create DynamoDB
ddb_table = _ddb.Table(self,
id='sa-table-cms',
table_name='sa_table_cms',
partition_key=_ddb.Attribute(
name='ID', type=_ddb.AttributeType.STRING))
# Permissions
lambda_role = _iam.Role(
self,
id='sa-cms-role',
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com'))
# WARNING: This is an example on how to include AWS managed policy. Don't use this for production. Please use the example of policy document for DynamoDB instead.
lambda_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
'CloudWatchLogsFullAccess'))
policy_statement = _iam.PolicyStatement(effect=_iam.Effect.ALLOW)
policy_statement.add_actions("dynamodb:*")
policy_statement.add_resources(ddb_table.table_arn)
lambda_role.add_to_policy(policy_statement)
# Create Lambda Functions
fn_lambda_create_post = _lambda.Function(
self,
"sa-cms-createPost",
code=_lambda.AssetCode("../sa-lambda/create-post/"),
handler="app.lambda_handler",
tracing=_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=_lambda.Runtime.PYTHON_3_8)
fn_lambda_list_post = _lambda.Function(
self,
"sa-cms-listPost",
code=_lambda.AssetCode("../sa-lambda/list-post/"),
handler="app.lambda_handler",
tracing=_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=_lambda.Runtime.PYTHON_3_8)
fn_lambda_get_post = _lambda.Function(
self,
"sa-cms-getPost",
code=_lambda.AssetCode("../sa-lambda/get-post/"),
handler="app.lambda_handler",
tracing=_lambda.Tracing.ACTIVE,
timeout=core.Duration.seconds(30),
role=lambda_role,
runtime=_lambda.Runtime.PYTHON_3_8)
fn_lambda_delete_post = _lambda.Function(
self,
"sa-cms-deletePost",
code=_lambda.AssetCode("../sa-lambda/delete-post/"),
handler="app.lambda_handler",
timeout=core.Duration.seconds(30),
tracing=_lambda.Tracing.ACTIVE,
role=lambda_role,
runtime=_lambda.Runtime.PYTHON_3_8)
fn_lambda_create_post.add_environment("DYNAMODB_TABLE",
ddb_table.table_name)
fn_lambda_get_post.add_environment("DYNAMODB_TABLE",
ddb_table.table_name)
fn_lambda_list_post.add_environment("DYNAMODB_TABLE",
ddb_table.table_name)
fn_lambda_delete_post.add_environment("DYNAMODB_TABLE",
ddb_table.table_name)
api = _ag.RestApi(
self,
id='sa-api-gateway',
# default_cors_preflight_options=_ag.CorsOptions(
# allow_methods=['ANY'],
# allow_origins=['*'],
# allow_headers=['Access-Control-Allow-Origin','Access-Control-Allow-Headers','Content-Type']
# )
)
posts_resource = api.root.add_resource('posts')
posts_with_id_resource = posts_resource.add_resource("{id}")
posts_resource.add_method('POST',
_ag.LambdaIntegration(fn_lambda_create_post))
posts_with_id_resource.add_method(
'GET', _ag.LambdaIntegration(fn_lambda_get_post))
posts_resource.add_method('GET',
_ag.LambdaIntegration(fn_lambda_list_post))
posts_with_id_resource.add_method(
'DELETE', _ag.LambdaIntegration(fn_lambda_delete_post))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
alexa_assets = os.path.dirname(
os.path.realpath(__file__)) + "/../skill"
asset = assets.Asset(self, 'SkillAsset', path=alexa_assets)
# role to access bucket
role = Role(self,
'Role',
assumed_by=CompositePrincipal(
ServicePrincipal('alexa-appkit.amazon.com'),
ServicePrincipal('cloudformation.amazonaws.com')))
# Allow the skill resource to access the zipped skill package
role.add_to_policy(
PolicyStatement(
actions=['S3:GetObject'],
resources=[
f'arn:aws:s3:::{asset.s3_bucket_name}/{asset.s3_object_key}'
]))
# DynamoDB Table
users_table = dynamo_db.Table(
self,
'Users',
partition_key=dynamo_db.Attribute(
name='userId', type=dynamo_db.AttributeType.STRING),
billing_mode=dynamo_db.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY)
# install node dependencies for lambdas
lambda_folder = os.path.dirname(
os.path.realpath(__file__)) + "/../lambda_fns"
subprocess.check_call("npm i".split(), cwd=lambda_folder)
subprocess.check_call("npm run build".split(), cwd=lambda_folder)
alexa_lambda = _lambda.Function(
self,
"AlexaLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
code=_lambda.Code.from_asset("lambda_fns"),
handler="lambda.handler",
environment={"USERS_TABLE": users_table.table_name})
# grant the lambda role read/write permissions to our table
users_table.grant_read_write_data(alexa_lambda)
# create the skill
skill = alexa_ask.CfnSkill(
self,
'the-alexa-skill',
vendor_id='',
authentication_configuration={
'clientId': '',
'clientSecret': '',
'refreshToken': ''
},
skill_package={
's3Bucket': asset.s3_bucket_name,
's3Key': asset.s3_object_key,
's3BucketRole': role.role_arn,
'overrides': {
'manifest': {
'apis': {
'custom': {
'endpoint': {
'uri': alexa_lambda.function_arn
}
}
}
}
}
})
###
# Allow the Alexa service to invoke the fulfillment Lambda.
# In order for the Skill to be created, the fulfillment Lambda
# must have a permission allowing Alexa to invoke it, this causes
# a circular dependency and requires the first deploy to allow all
# Alexa skills to invoke the lambda, subsequent deploys will work
# when specifying the eventSourceToken
###
alexa_lambda.add_permission(
'AlexaPermission',
# eventSourceToken: skill.ref,
principal=ServicePrincipal('alexa-appkit.amazon.com'),
action='lambda:InvokeFunction')
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# ugly, but we need to read user_pool_id from amplify's aws-exports.js file
with open('../aws-exports.js') as dataFile:
data = dataFile.read()
obj = data[data.find('{'):data.rfind('}') + 1]
aws_exports = json.loads(obj)
user_pool_id = aws_exports["aws_user_pools_id"]
# use that user_pool_id to define a cdk struct representing it
user_pool = aws_cognito.UserPool.from_user_pool_id(
self, 'amplify_user_pool', user_pool_id)
api = aws_appsync.GraphqlApi(
self,
'ring-it-up-api',
name='ring-it-up-api',
schema=aws_appsync.Schema.from_asset('graphql/schema.graphql'),
authorization_config=aws_appsync.AuthorizationConfig(
default_authorization=aws_appsync.AuthorizationMode(
authorization_type=aws_appsync.AuthorizationType.API_KEY,
api_key_config=aws_appsync.ApiKeyConfig(
expires=core.Expiration.after(core.Duration.days(
365)))),
additional_authorization_modes=[
aws_appsync.AuthorizationMode(
authorization_type=aws_appsync.AuthorizationType.
USER_POOL,
user_pool_config=aws_appsync.UserPoolConfig(
user_pool=user_pool))
]))
# Prints out the AppSync GraphQL API URL to the ternminal
core.CfnOutput(self,
"aws_appsync_graphqlEndpoint",
value=api.graphql_url)
# Prints out the AppSync GraphQL API key to the terminal
core.CfnOutput(self, "aws_appsync_apiKey", value=api.api_key)
# Prints out the base authentication type for API
core.CfnOutput(self,
"aws_appsync_authenticationType",
value=str(aws_appsync.AuthorizationType.API_KEY))
api_lambda = aws_lambda.Function(
self,
'AppSyncBlogHandler',
runtime=aws_lambda.Runtime.NODEJS_12_X,
handler='main.handler',
code=aws_lambda.Code.from_asset('lambda-fns'),
memory_size=1024)
# Set the new Lambda function as a data source for the AppSync API
lambda_datasource = api.add_lambda_data_source('lambdaDatasource',
api_lambda)
lambda_datasource.create_resolver(type_name="Query",
field_name="getActivityById")
lambda_datasource.create_resolver(type_name="Query",
field_name="listActivities")
lambda_datasource.create_resolver(type_name="Mutation",
field_name="createActivity")
lambda_datasource.create_resolver(type_name="Mutation",
field_name="deleteActivity")
lambda_datasource.create_resolver(type_name="Mutation",
field_name="updateActivity")
activity_table = aws_dynamodb.Table(
self,
'CDKPostTable',
removal_policy=core.RemovalPolicy.DESTROY,
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
partition_key={
"name": 'id',
"type": aws_dynamodb.AttributeType.STRING,
})
# enable the Lambda function to access the DynamoDB table (using IAM)
activity_table.grant_full_access(api_lambda)
# Create an environment variable that we will use in the function code
api_lambda.add_environment('ACTIVITY_TABLE', activity_table.table_name)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
################################################################################
# VPC
vpc = ec2.Vpc(self, "Monitoring VPC", max_azs=3)
################################################################################
# Amazon OpenSearch Service domain
es_sec_grp = ec2.SecurityGroup(
self,
'OpenSearchSecGrpMonitoring',
vpc=vpc,
allow_all_outbound=True,
security_group_name='OpenSearchSecGrpMonitoring')
es_sec_grp.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(80))
es_sec_grp.add_ingress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(443))
domain = opensearch.Domain(
self,
'opensearch-service-monitor',
version=opensearch.EngineVersion.
OPENSEARCH_1_0, # Upgrade when CDK upgrades
domain_name=DOMAIN_NAME,
removal_policy=core.RemovalPolicy.DESTROY,
capacity=opensearch.CapacityConfig(
data_node_instance_type=DOMAIN_DATA_NODE_INSTANCE_TYPE,
data_nodes=DOMAIN_DATA_NODE_INSTANCE_COUNT,
master_node_instance_type=DOMAIN_MASTER_NODE_INSTANCE_TYPE,
master_nodes=DOMAIN_MASTER_NODE_INSTANCE_COUNT,
warm_instance_type=DOMAIN_UW_NODE_INSTANCE_TYPE,
warm_nodes=DOMAIN_UW_NODE_INSTANCE_COUNT),
ebs=opensearch.EbsOptions(enabled=True,
volume_size=DOMAIN_INSTANCE_VOLUME_SIZE,
volume_type=ec2.EbsDeviceVolumeType.GP2),
vpc=vpc,
vpc_subnets=[ec2.SubnetType.PUBLIC],
security_groups=[es_sec_grp],
zone_awareness=opensearch.ZoneAwarenessConfig(
enabled=True, availability_zone_count=DOMAIN_AZ_COUNT),
enforce_https=True,
node_to_node_encryption=True,
encryption_at_rest={"enabled": True},
use_unsigned_basic_auth=True,
fine_grained_access_control={
"master_user_name":
DOMAIN_ADMIN_UNAME,
"master_user_password":
core.SecretValue.plain_text(DOMAIN_ADMIN_PW)
})
core.CfnOutput(
self,
"MasterUser",
value=DOMAIN_ADMIN_UNAME,
description="Master User Name for Amazon OpenSearch Service")
core.CfnOutput(
self,
"MasterPW",
value=DOMAIN_ADMIN_PW,
description="Master User Password for Amazon OpenSearch Service")
################################################################################
# Dynamo DB table for time stamp tracking
table = ddb.Table(
self,
'opensearch-monitor-lambda-timestamp',
table_name=TABLE_NAME,
partition_key=ddb.Attribute(name="domain",
type=ddb.AttributeType.STRING),
sort_key=ddb.Attribute(name='region',
type=ddb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.DESTROY)
################################################################################
# Lambda monitoring function
lambda_func = lambda_.Function(
self,
'CWMetricsToOpenSearch',
function_name="CWMetricsToOpenSearch_monitoring",
runtime=lambda_.Runtime.PYTHON_3_8,
code=lambda_.Code.asset('CWMetricsToOpenSearch'),
handler='handler.handler',
memory_size=1024,
timeout=core.Duration.minutes(10),
vpc=vpc)
table.grant_read_data(lambda_func)
table.grant_write_data(lambda_func)
lambda_func.add_environment('TABLE', table.table_name)
lambda_func.add_environment('DOMAIN_ENDPOINT',
'https://' + domain.domain_endpoint)
lambda_func.add_environment('DOMAIN_ADMIN_UNAME', DOMAIN_ADMIN_UNAME)
lambda_func.add_environment('DOMAIN_ADMIN_PW', DOMAIN_ADMIN_PW)
lambda_func.add_environment('REGIONS', REGIONS_TO_MONITOR)
# When the domain is created here, restrict access
lambda_func.add_to_role_policy(
iam.PolicyStatement(actions=['es:*'], resources=['*']))
# The function needs to read CW events. Restrict
lambda_func.add_to_role_policy(
iam.PolicyStatement(actions=['cloudwatch:*'], resources=['*']))
lambda_schedule = events.Schedule.rate(
core.Duration.seconds(LAMBDA_INTERVAL))
event_lambda_target = targets.LambdaFunction(handler=lambda_func)
events.Rule(self,
"Monitoring",
enabled=True,
schedule=lambda_schedule,
targets=[event_lambda_target])
################################################################################
# Lambda for CW Logs
lambda_func_cw_logs = lambda_.Function(
self,
'CWLogsToOpenSearch',
function_name="CWLogsToOpenSearch_monitoring",
runtime=lambda_.Runtime.NODEJS_12_X,
code=lambda_.Code.asset('CWLogsToOpenSearch'),
handler='index.handler',
vpc=vpc)
# # Load Amazon OpenSearch Service Domain to env variable
lambda_func_cw_logs.add_environment('DOMAIN_ENDPOINT',
domain.domain_endpoint)
# # When the domain is created here, restrict access
lambda_func_cw_logs.add_to_role_policy(
iam.PolicyStatement(actions=['es:*'], resources=['*']))
# # The function needs to read CW Logs. Restrict
lambda_func_cw_logs.add_to_role_policy(
iam.PolicyStatement(actions=['logs:*'], resources=['*']))
# Add permission to create CW logs trigger for all specified region and current account, as region does not have an option to be wildcard
account_id = boto3.client("sts").get_caller_identity()["Account"]
for region in json.loads(REGIONS_TO_MONITOR):
lambda_func_cw_logs.add_permission(
id="lambda-cw-logs-permission-" + region,
principal=iam.ServicePrincipal("logs.amazonaws.com"),
action="lambda:InvokeFunction",
source_arn="arn:aws:logs:" + region + ":" + account_id +
":*:*:*")
################################################################################
# Jump host for SSH tunneling and direct access
sn_public = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC)
amzn_linux = ec2.MachineImage.latest_amazon_linux(
generation=ec2.AmazonLinuxGeneration.AMAZON_LINUX_2,
edition=ec2.AmazonLinuxEdition.STANDARD,
virtualization=ec2.AmazonLinuxVirt.HVM,
storage=ec2.AmazonLinuxStorage.GENERAL_PURPOSE)
# Instance Role and SSM Managed Policy
role = iam.Role(self,
"InstanceSSM",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"))
role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AmazonEC2RoleforSSM"))
role.add_managed_policy(
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonSSMManagedInstanceCore"))
instance = ec2.Instance(
self,
'instance',
instance_type=ec2.InstanceType(EC2_INSTANCE_TYPE),
vpc=vpc,
machine_image=amzn_linux,
vpc_subnets=sn_public,
key_name=EC2_KEY_NAME,
role=role,
)
instance.connections.allow_from_any_ipv4(ec2.Port.tcp(22), 'SSH')
instance.connections.allow_from_any_ipv4(ec2.Port.tcp(443), 'HTTPS')
stmt = iam.PolicyStatement(actions=['es:*'],
resources=[domain.domain_arn])
instance.add_to_role_policy(stmt)
# Create SNS topic, subscription, IAM roles, Policies
sns_topic = sns.Topic(self, "cdk_monitoring_topic")
sns_topic.add_subscription(
subscriptions.EmailSubscription(SNS_NOTIFICATION_EMAIL))
sns_policy_statement = iam.PolicyStatement(
actions=["sns:publish"],
resources=[sns_topic.topic_arn],
effect=iam.Effect.ALLOW)
sns_policy = iam.ManagedPolicy(self, "cdk_monitoring_policy")
sns_policy.add_statements(sns_policy_statement)
sns_role = iam.Role(
self,
"cdk_monitoring_sns_role",
assumed_by=iam.ServicePrincipal("es.amazonaws.com"))
sns_role.add_managed_policy(sns_policy)
dirname = os.path.dirname(__file__)
dashboards_asset = Asset(
self,
"DashboardsAsset",
path=os.path.join(dirname,
'export_opensearch_dashboards_V1_0.ndjson'))
dashboards_asset.grant_read(instance.role)
dashboards_asset_path = instance.user_data.add_s3_download_command(
bucket=dashboards_asset.bucket,
bucket_key=dashboards_asset.s3_object_key,
)
nginx_asset = Asset(self,
"NginxAsset",
path=os.path.join(dirname,
'nginx_opensearch.conf'))
nginx_asset.grant_read(instance.role)
nginx_asset_path = instance.user_data.add_s3_download_command(
bucket=nginx_asset.bucket,
bucket_key=nginx_asset.s3_object_key,
)
alerting_asset = Asset(self,
"AlertingAsset",
path=os.path.join(dirname, 'create_alerts.sh'))
alerting_asset.grant_read(instance.role)
alerting_asset_path = instance.user_data.add_s3_download_command(
bucket=alerting_asset.bucket,
bucket_key=alerting_asset.s3_object_key,
)
instance.user_data.add_commands(
"yum update -y",
"yum install jq -y",
"amazon-linux-extras install nginx1.12",
"cd /tmp/assets",
"mv {} export_opensearch_dashboards_V1_0.ndjson".format(
dashboards_asset_path),
"mv {} nginx_opensearch.conf".format(nginx_asset_path),
"mv {} create_alerts.sh".format(alerting_asset_path),
"openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/nginx/cert.key -out /etc/nginx/cert.crt -subj /C=US/ST=./L=./O=./CN=.\n"
"cp nginx_opensearch.conf /etc/nginx/conf.d/",
"sed -i 's/DEFAULT_DOMAIN_NAME/" + DOMAIN_NAME +
"/g' /tmp/assets/export_opensearch_dashboards_V1_0.ndjson",
"sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint +
"/g' /etc/nginx/conf.d/nginx_opensearch.conf",
"sed -i 's/DOMAIN_ENDPOINT/" + domain.domain_endpoint +
"/g' /tmp/assets/create_alerts.sh",
"sed -i 's=LAMBDA_CW_LOGS_ROLE_ARN=" +
lambda_func_cw_logs.role.role_arn +
"=g' /tmp/assets/create_alerts.sh",
"sed -i 's=SNS_ROLE_ARN=" + sns_role.role_arn +
"=g' /tmp/assets/create_alerts.sh",
"sed -i 's/SNS_TOPIC_ARN/" + sns_topic.topic_arn +
"/g' /tmp/assets/create_alerts.sh",
"sed -i 's=DOMAIN_ADMIN_UNAME=" + DOMAIN_ADMIN_UNAME +
"=g' /tmp/assets/create_alerts.sh",
"sed -i 's=DOMAIN_ADMIN_PW=" + DOMAIN_ADMIN_PW +
"=g' /tmp/assets/create_alerts.sh",
"systemctl restart nginx.service",
"chmod 500 create_alerts.sh",
"sleep 5",
"bash --verbose create_alerts.sh",
)
core.CfnOutput(self,
"Dashboards URL (via Jump host)",
value="https://" + instance.instance_public_ip,
description="Dashboards URL via Jump host")
core.CfnOutput(
self,
"SNS Subscription Alert Message",
value=SNS_NOTIFICATION_EMAIL,
description="Please confirm your SNS subscription receievedt at")
def __init__(self, scope: core.Construct, id: str, vpc_id: str, subnet_ids, rds_secret_arn: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
vpc = ec2.Vpc.from_vpc_attributes(self, vpc_id, vpc_id=vpc_id,
availability_zones= [ 'eu-west-1c'],
public_subnet_ids= subnet_ids)
# creating vote table for dynamodb resolver
vote_table = ddb.Table(self, 'votes',
table_name='votes',
partition_key={
"name": "productid",
"type": ddb.AttributeType.STRING
},
# Sortkey structure is like : UP#20200902T12:34:00 - DOWN#20201030T10:45:12
sort_key={
"name": "votesortkey",
"type": ddb.AttributeType.STRING
},
read_capacity=5,
write_capacity=5
)
# creating API with GraphQL schema
api = appsync.GraphqlApi(self, 'example_appsync_api',
name="example_appsync_api",
log_config=appsync.LogConfig(field_log_level=appsync.FieldLogLevel.ALL),
schema=appsync.Schema.from_asset(file_path="../appsync-conf/schema.graphql")
)
# Authentication done with API key - for development purposes only
appsync.CfnApiKey(self, 'examplegraphqlapi',
api_id=api.api_id
)
# create security group for lambda
# this will need to be added to your RDS inbound
lambda_security_group = ec2.SecurityGroup(self, "Example-AppSyncResolverLambdaSG",
security_group_name="Example-AppSyncResolverLambdaSG",
vpc=vpc,
allow_all_outbound=True
)
# getting the code from local directory
lambda_rds_code = aws_lambda.Code.asset("../lambda-rds")
lambda_rds_resolver = aws_lambda.Function(self,
"LambdaAppSyncSQLResolver",
function_name=f"LambdaAppSyncSQLResolver",
code=lambda_rds_code,
handler="index.handler",
runtime=aws_lambda.Runtime.NODEJS_12_X,
memory_size=512,
timeout=core.Duration.seconds(60),
log_retention=logs.RetentionDays.ONE_MONTH,
vpc=vpc,
vpc_subnets={
"subnet_type": ec2.SubnetType.PUBLIC
},
allow_public_subnet=True,
security_group=lambda_security_group,
)
# env parameters for rds lambda to perform SQL calls
lambda_rds_resolver.add_environment("SECRET_ARN", rds_secret_arn)
# allow lambda to read secret
lambda_rds_resolver.add_to_role_policy(iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[ 'secretsmanager:GetSecretValue' ],
resources=[ rds_secret_arn ]
))
# adding the product datasource as lamda resolver
products_ds = api.add_lambda_data_source('Products', lambda_rds_resolver)
# creates resolver for query getProduct
products_ds.create_resolver(
type_name='Query',
field_name='getProduct',
request_mapping_template=appsync.MappingTemplate.from_file("../appsync-conf/vtl/getProduct.vtl"),
response_mapping_template=appsync.MappingTemplate.from_file("../appsync-conf/vtl/getProduct_output_template.vtl"),
)
# adding lamda resolver for vote fields in product model
lambda_dynamodb_code = aws_lambda.Code.asset("../lambda-dynamodb")
lambda_dynamodb_votes_resolver = aws_lambda.Function(self,
"LambdaAppSyncVotesResolver",
function_name=f"LambdaAppSyncVotesResolver",
code=lambda_dynamodb_code,
handler="index.handler",
runtime=aws_lambda.Runtime.NODEJS_12_X,
memory_size=512,
timeout=core.Duration.seconds(60),
)
# allow lambda to query dynamodb
lambda_dynamodb_votes_resolver.add_to_role_policy(iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"dynamodb:GetItem",
"dynamodb:Query",
],
resources=[
vote_table.table_arn,
vote_table.table_arn + "/*"
]
));
# create lambda datasource for dynamodb queries
votes_ds = api.add_lambda_data_source('Votes', lambda_dynamodb_votes_resolver)
votes_ds.create_resolver(
type_name='Product',
field_name='ups',
request_mapping_template=appsync.MappingTemplate.from_file("../appsync-conf/vtl/fields/votes_up.vtl"),
response_mapping_template=appsync.MappingTemplate.from_file("../appsync-conf/vtl/fields/votes_up_output_template.vtl"),
)
votes_ds.create_resolver(
type_name='Product',
field_name='downs',
request_mapping_template=appsync.MappingTemplate.from_file("../appsync-conf/vtl/fields/votes_down.vtl"),
response_mapping_template=appsync.MappingTemplate.from_file("../appsync-conf/vtl/fields/votes_down_output_template.vtl"),
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# DynamoDB Table
# This will store our error records
# TTL Docs - https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/time-to-live-ttl-how-to.html
table = dynamo_db.Table(self,
"CircuitBreaker",
partition_key=dynamo_db.Attribute(
name="RequestID",
type=dynamo_db.AttributeType.STRING),
sort_key=dynamo_db.Attribute(
name="ExpirationTime",
type=dynamo_db.AttributeType.NUMBER),
time_to_live_attribute='ExpirationTime')
# Add an index that lets us query on site url and Expiration Time
table.add_global_secondary_index(
index_name='UrlIndex',
partition_key=dynamo_db.Attribute(
name="SiteUrl", type=dynamo_db.AttributeType.STRING),
sort_key=dynamo_db.Attribute(name="ExpirationTime",
type=dynamo_db.AttributeType.NUMBER))
# defines an Integration Lambda to call our failing web service
integration_lambda = _lambda.Function(
self,
"WebserviceIntegrationLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="lambda.handler",
code=_lambda.Code.from_asset("lambda_fns/webservice"),
timeout=core.Duration.seconds(20),
environment=dict(TABLE_NAME=table.table_name))
# grant the lambda role read/write permissions to our table
table.grant_read_data(integration_lambda)
# We need to give your lambda permission to put events on our EventBridge
event_policy = iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=['*'],
actions=['events:PutEvents'])
integration_lambda.add_to_role_policy(event_policy)
# defines a lambda to insert errors into dynamoDB
error_lambda = _lambda.Function(
self,
"ErrorLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="lambda.handler",
code=_lambda.Code.from_asset("lambda_fns/error"),
timeout=core.Duration.seconds(3),
environment=dict(TABLE_NAME=table.table_name))
table.grant_write_data(error_lambda)
# Create EventBridge rule to route failures
error_rule = events.Rule(
self,
'webserviceErrorRule',
description='Failed Webservice Call',
event_pattern=events.EventPattern(
source=['cdkpatterns.eventbridge.circuitbreaker'],
detail_type=['httpcall'],
detail={"status": ["fail"]}))
error_rule.add_target(targets.LambdaFunction(handler=error_lambda))
# defines an API Gateway REST API resource backed by our "integration_lambda" function
api_gw.LambdaRestApi(self,
'CircuitBreakerGateway',
handler=integration_lambda)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# create dynamo table
template_table = aws_dynamodb.Table(
self,
TABLE_TEMPLATE_NAME,
table_name=TABLE_TEMPLATE_NAME,
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.
DESTROY, #开发阶段设置为 DESTROY， 正式环境设置为.RETAIN
partition_key=aws_dynamodb.Attribute(
name="template_type", type=aws_dynamodb.AttributeType.STRING),
sort_key=aws_dynamodb.Attribute(
name='create_time', type=aws_dynamodb.AttributeType.STRING))
field_table = aws_dynamodb.Table(
self,
TABLE_FIELD_NAME,
table_name=TABLE_FIELD_NAME,
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.
DESTROY, #开发阶段设置为 DESTROY， 正式环境设置为.RETAIN
partition_key=aws_dynamodb.Attribute(
name="template_id", type=aws_dynamodb.AttributeType.STRING),
sort_key=aws_dynamodb.Attribute(
name='business_field', type=aws_dynamodb.AttributeType.STRING))
# create producer lambda function
producer_lambda = aws_lambda.Function(
self,
"producer_lambda_function",
function_name="OcrData",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="lambda_function.lambda_handler",
code=aws_lambda.Code.asset("./lambda/producer"))
producer_lambda.add_environment("TABLE_TEMPLATE_NAME",
template_table.table_name)
producer_lambda.add_environment("TABLE_FIELD_NAME",
field_table.table_name)
template_table.grant_read_write_data(producer_lambda)
field_table.grant_read_write_data(producer_lambda)
# ---------------------api-gateway GET -------------------- #
integration_responses = [{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': "'*'",
}
}]
method_responses = [{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}]
producer_api = aws_apigateway.RestApi(
self,
'OcrApi',
endpoint_types=[aws_apigateway.EndpointType.REGIONAL],
rest_api_name='OcrApi')
method_post_data = "ocr"
producer_entity = producer_api.root.add_resource(method_post_data)
producer_entity_lambda_integration = aws_apigateway.LambdaIntegration(
producer_lambda,
proxy=False,
integration_responses=integration_responses)
producer_entity.add_method('POST',
producer_entity_lambda_integration,
method_responses=method_responses)
core.CfnOutput(self,
"OCR-Data",
value=producer_api.url + method_post_data,
description="ocr url")
self.add_cors_options(producer_entity)
def __init__(self,
scope: cdk.Stack,
id: str,
git_hash,
sla_zip='sla-monitor.zip',
**kwargs):
super().__init__(scope, id, **kwargs)
self.sla_zip = git_hash + "-" + sla_zip
self.sla_monitor_dynamo_table = aws_dynamodb.Table(
self,
"DynamoTable{}".format("SLAMonitor"),
table_name=self.stack_name,
billing_mode=aws_dynamodb.BillingMode.PayPerRequest,
stream_specification=aws_dynamodb.StreamViewType.NewImage,
partition_key={
"name": "service_name",
"type": aws_dynamodb.AttributeType.String
},
sort_key={
"name": "last_updated_date",
"type": aws_dynamodb.AttributeType.String
},
)
self.sla_monitor_lambda_function = aws_lambda.Function(
self,
"SLAMonitorLambdaFunction",
function_name=self.stack_name,
code=aws_lambda.AssetCode(self.sla_zip),
handler="main.lambda_handler",
runtime=aws_lambda.Runtime.PYTHON37,
tracing=aws_lambda.Tracing.Active,
#layers=[self.dynamodb_lambda_layer],
description=
"Monitors AWS SLA Pages and updates DynamoDB Table when SLAs update",
environment={
"STACK_NAME": self.stack_name,
"LOCAL_MODE": "False",
"DYNAMO_TABLE_NAME": self.stack_name
},
memory_size=128,
timeout=90,
)
# Permissions to access sla_monitor dynamo table
self.sla_monitor_dynamo_table.grant_read_write_data(
self.sla_monitor_lambda_function.role)
self.sla_monitor_cw_event_rule = aws_events.Rule(
self,
"LambdaCWEventRuleSLAMonitor",
description="Scheduled event to trigger AWS SLA monitor",
enabled=True,
#schedule_expression='cron(0 22 */3 * ? *)',
schedule_expression='cron(*/8 * * * ? *)',
targets=[
aws_events_targets.LambdaFunction(
handler=self.sla_monitor_lambda_function)
],
)
def __init__(self, scope: core.App, name: str, **kwargs) -> None:
super().__init__(scope, name, **kwargs)
# <1>
# dynamoDB table to store questions and answers
table = dynamodb.Table(
self,
"EcsClusterQaBot-Table",
partition_key=dynamodb.Attribute(
name="item_id", type=dynamodb.AttributeType.STRING),
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY)
# <2>
vpc = ec2.Vpc(
self,
"EcsClusterQaBot-Vpc",
max_azs=1,
subnet_configuration=[
ec2.SubnetConfiguration(
name="public",
subnet_type=ec2.SubnetType.PUBLIC,
)
],
nat_gateways=0,
)
# <3>
cluster = ecs.Cluster(
self,
"EcsClusterQaBot-Cluster",
vpc=vpc,
)
# <4>
taskdef = ecs.FargateTaskDefinition(
self,
"EcsClusterQaBot-TaskDef",
cpu=1024, # 1 CPU
memory_limit_mib=4096, # 4GB RAM
)
# grant permissions
table.grant_read_write_data(taskdef.task_role)
taskdef.add_to_task_role_policy(
iam.PolicyStatement(effect=iam.Effect.ALLOW,
resources=["*"],
actions=["ssm:GetParameter"]))
# <5>
container = taskdef.add_container(
"EcsClusterQaBot-Container",
image=ecs.ContainerImage.from_registry(
"registry.gitlab.com/tomomano/intro-aws/handson03:latest"),
logging=ecs.LogDrivers.aws_logs(
stream_prefix="EcsClusterQaBot",
log_retention=aws_logs.RetentionDays.ONE_DAY),
)
# Store parameters in SSM
ssm.StringParameter(
self,
"ECS_CLUSTER_NAME",
parameter_name="ECS_CLUSTER_NAME",
string_value=cluster.cluster_name,
)
ssm.StringParameter(self,
"ECS_TASK_DEFINITION_ARN",
parameter_name="ECS_TASK_DEFINITION_ARN",
string_value=taskdef.task_definition_arn)
ssm.StringParameter(self,
"ECS_TASK_VPC_SUBNET_1",
parameter_name="ECS_TASK_VPC_SUBNET_1",
string_value=vpc.public_subnets[0].subnet_id)
ssm.StringParameter(self,
"CONTAINER_NAME",
parameter_name="CONTAINER_NAME",
string_value=container.container_name)
ssm.StringParameter(self,
"TABLE_NAME",
parameter_name="TABLE_NAME",
string_value=table.table_name)
core.CfnOutput(self, "ClusterName", value=cluster.cluster_name)
core.CfnOutput(self,
"TaskDefinitionArn",
value=taskdef.task_definition_arn)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# DDB table to store the Long and Short URLs with Short URL as the partition key
url_mapping_table = ddb.Table(
self,
"url_shortener_mapping_table",
partition_key=ddb.Attribute(name="short_url",
type=ddb.AttributeType.STRING),
read_capacity=10,
write_capacity=10,
removal_policy=core.RemovalPolicy.DESTROY,
)
# AutoScaling of RCUs with a Target Utilization of 70%
url_mapping_table.auto_scale_read_capacity(
min_capacity=10, max_capacity=40000).scale_on_utilization(
target_utilization_percent=70)
# AutoScaling of WCUs with a Target Utilization of 70%
url_mapping_table.auto_scale_write_capacity(
min_capacity=10, max_capacity=40000).scale_on_utilization(
target_utilization_percent=70)
# DDB table to keep track of an Atomic Counter used for generating Short URLs
url_counter_table = ddb.Table(
self,
"url_shortener_counter_table",
partition_key=ddb.Attribute(name="id",
type=ddb.AttributeType.STRING),
read_capacity=10,
write_capacity=10,
removal_policy=core.RemovalPolicy.DESTROY,
)
# AutoScaling of RCUs with a Target Utilization of 70%
url_counter_table.auto_scale_read_capacity(
min_capacity=10, max_capacity=40000).scale_on_utilization(
target_utilization_percent=70)
# AutoScaling of WCUs with a Target Utilization of 70%
url_counter_table.auto_scale_write_capacity(
min_capacity=10, max_capacity=40000).scale_on_utilization(
target_utilization_percent=70)
# Lambda function with custom code to handle shortening/unshortening logic
url_lambda = _lambda.Function(
self,
"url_shortener_lambda",
code=_lambda.Code.asset("lambda_proxy"),
handler="lambda_function.lambda_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
timeout=core.Duration.seconds(10),
environment={
"BACKOFF": "25",
"HASH_DIGEST_SIZE": "8",
"MAX_RETRIES": "3",
"URL_SHORTENER_MAPPING_TABLE": url_mapping_table.table_name,
"URL_SHORTENER_COUNTER_TABLE": url_counter_table.table_name,
},
log_retention=logs.RetentionDays.ONE_MONTH,
)
# A Custom IAM Policy statement to grant DDB access to the Lambda function
ddb_policy_statement = iam.PolicyStatement(
actions=[
"dynamodb:PutItem", "dynamodb:GetItem", "dynamodb:UpdateItem"
],
effect=iam.Effect.ALLOW,
resources=[
url_mapping_table.table_arn, url_counter_table.table_arn
],
)
# Attaching DDB Policy statement with the Lambda IAM Role
url_lambda.add_to_role_policy(ddb_policy_statement)
# Including X-Requested-With to the default CORS headers list
headers = apigw.Cors.DEFAULT_HEADERS
headers.append('X-Requested-With')
# API Gateway endpoint to serve Shorten/Unshorten APIs
url_rest_api = apigw.RestApi(
self,
"url_shortener_API",
default_cors_preflight_options=apigw.CorsOptions(
allow_origins=apigw.Cors.ALL_ORIGINS,
allow_headers=headers,
allow_methods=["POST", "GET", "OPTIONS"],
status_code=200,
),
)
# Shorten API using POST and Lambda proxy
url_rest_api.root.add_resource(path_part="shorten", ).add_method(
http_method="POST",
request_models={
"application/json": apigw.Model.EMPTY_MODEL,
},
integration=apigw.LambdaIntegration(
handler=url_lambda,
proxy=True,
allow_test_invoke=True,
),
)
# Unshorten API using GET and Lambda proxy
url_rest_api.root.add_resource(path_part="unshorten", ).add_resource(
path_part="{shorturl}").add_method(
http_method="GET",
request_models={
"application/json": apigw.Model.EMPTY_MODEL,
},
integration=apigw.LambdaIntegration(
handler=url_lambda,
proxy=True,
allow_test_invoke=True,
),
)
# S3 bucket to host the URL Shortener Static Website
s3_web_hosting = s3.Bucket(
self,
"url_shortener_web_hosting_bucket",
website_index_document="index.html",
)
# Uploading HTML and ICO files from local directory to S3 Static Website bucket
s3_deploy = s3deploy.BucketDeployment(
self,
"website_source_files",
sources=[s3deploy.Source.asset(path="website", )],
destination_bucket=s3_web_hosting,
)
# Lambda function to integrate the API GW Shorten endpoint with the HTML file stored in S3
cr_provider = _lambda.Function(
self,
"cr_provider",
code=_lambda.Code.asset("custom_resource"),
handler="lambda_function.lambda_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
timeout=core.Duration.minutes(1),
)
# A Custom IAM Policy statement to grant S3 access to the Lambda function
lambda_cr_statement = iam.PolicyStatement(
actions=["s3:List*", "s3:Get*", "s3:Put*"],
effect=iam.Effect.ALLOW,
resources=[
s3_web_hosting.bucket_arn, s3_web_hosting.bucket_arn + "/*"
])
cr_provider.add_to_role_policy(lambda_cr_statement)
# CFN Custom Resource backed by Lambda
lambda_cr = core.CustomResource(
self,
"lambda_cr",
service_token=cr_provider.function_arn,
properties={
"S3_BUCKET": s3_web_hosting.bucket_name,
"S3_KEY": "index.html",
"POST_URL": url_rest_api.url + "shorten",
},
removal_policy=core.RemovalPolicy.DESTROY,
)
# Adding dependency so that Custom Resource creation happens after files are uploaded to S3
lambda_cr.node.add_dependency(s3_deploy)
# CloudFront Distribution with S3 and APIGateway origins
url_cf_distribution = cf.CloudFrontWebDistribution(
self,
"url_shortener_distribution",
origin_configs=[
cf.SourceConfiguration(s3_origin_source=cf.S3OriginConfig(
s3_bucket_source=s3_web_hosting,
origin_access_identity=cf.OriginAccessIdentity(
self,
id="OAI",
comment=
"OAI that allows CloudFront to access the S3 bucket"),
),
behaviors=[
cf.Behavior(
is_default_behavior=False,
path_pattern="/index.html",
),
cf.Behavior(
is_default_behavior=False,
path_pattern="/favicon.ico",
),
]),
cf.SourceConfiguration(
custom_origin_source=cf.CustomOriginConfig(
domain_name=url_rest_api.url.lstrip("https://").split(
"/")[0], ),
origin_path="/" +
url_rest_api.deployment_stage.stage_name + "/unshorten",
behaviors=[
cf.Behavior(
is_default_behavior=True,
allowed_methods=cf.CloudFrontAllowedMethods.
GET_HEAD_OPTIONS,
)
])
],
price_class=cf.PriceClass.PRICE_CLASS_ALL,
default_root_object="index.html",
)
# Adding the CloudFront Distribution endpoint to CFN Output
core.CfnOutput(
self,
"URLShortenerWebsite",
value=url_cf_distribution.domain_name,
)
def __init__(self, scope: core.Construct, _id: str, **kwargs) -> None:
super().__init__(scope, _id, **kwargs)
# Setup SSM parameter of credentials, bucket_para, ignore_list
ssm_credential_para = ssm.StringParameter.from_secure_string_parameter_attributes(
self,
"ssm_parameter_credentials",
parameter_name=ssm_parameter_credentials,
version=1)
ssm_bucket_para = ssm.StringParameter(self,
"s3bucket_serverless",
string_value=json.dumps(
bucket_para, indent=4))
ssm_parameter_ignore_list = ssm.StringParameter(
self, "s3_migrate_ignore_list", string_value=ignore_list)
# Setup DynamoDB
ddb_file_list = ddb.Table(self,
"s3migrate_serverless",
partition_key=ddb.Attribute(
name="Key",
type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
ddb_file_list.add_global_secondary_index(
partition_key=ddb.Attribute(name="desBucket",
type=ddb.AttributeType.STRING),
index_name="desBucket-index",
projection_type=ddb.ProjectionType.INCLUDE,
non_key_attributes=["desKey", "versionId"])
# Setup SQS
sqs_queue_DLQ = sqs.Queue(self,
"s3migrate_serverless_Q_DLQ",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14))
sqs_queue = sqs.Queue(self,
"s3migrate_serverless_Q",
visibility_timeout=core.Duration.minutes(15),
retention_period=core.Duration.days(14),
dead_letter_queue=sqs.DeadLetterQueue(
max_receive_count=60, queue=sqs_queue_DLQ))
# Setup API for Lambda to get IP address (for debug networking routing purpose)
checkip = api.RestApi(
self,
"lambda-checkip-api",
cloud_watch_role=True,
deploy=True,
description="For Lambda get IP address",
default_integration=api.MockIntegration(
integration_responses=[
api.IntegrationResponse(status_code="200",
response_templates={
"application/json":
"$context.identity.sourceIp"
})
],
request_templates={"application/json": '{"statusCode": 200}'}),
endpoint_types=[api.EndpointType.REGIONAL])
checkip.root.add_method("GET",
method_responses=[
api.MethodResponse(
status_code="200",
response_models={
"application/json":
api.Model.EMPTY_MODEL
})
])
# Setup Lambda functions
handler = lam.Function(self,
"s3-migrate-worker",
code=lam.Code.asset("./lambda"),
handler="lambda_function_worker.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name':
ddb_file_list.table_name,
'Des_bucket_default': Des_bucket_default,
'Des_prefix_default': Des_prefix_default,
'StorageClass': StorageClass,
'checkip_url': checkip.url,
'ssm_parameter_credentials':
ssm_parameter_credentials,
'JobType': JobType,
'MaxRetry': MaxRetry,
'MaxThread': MaxThread,
'MaxParallelFile': MaxParallelFile,
'JobTimeout': JobTimeout,
'UpdateVersionId': UpdateVersionId,
'GetObjectWithVersionId':
GetObjectWithVersionId
})
handler_jobsender = lam.Function(
self,
"s3-migrate-jobsender",
code=lam.Code.asset("./lambda"),
handler="lambda_function_jobsender.lambda_handler",
runtime=lam.Runtime.PYTHON_3_8,
memory_size=1024,
timeout=core.Duration.minutes(15),
tracing=lam.Tracing.ACTIVE,
environment={
'table_queue_name': ddb_file_list.table_name,
'StorageClass': StorageClass,
'checkip_url': checkip.url,
'sqs_queue': sqs_queue.queue_name,
'ssm_parameter_credentials': ssm_parameter_credentials,
'ssm_parameter_ignore_list':
ssm_parameter_ignore_list.parameter_name,
'ssm_parameter_bucket': ssm_bucket_para.parameter_name,
'JobType': JobType,
'MaxRetry': MaxRetry,
'JobsenderCompareVersionId': JobsenderCompareVersionId
})
# Allow lambda read/write DDB, SQS
ddb_file_list.grant_read_write_data(handler)
ddb_file_list.grant_read_write_data(handler_jobsender)
sqs_queue.grant_send_messages(handler_jobsender)
# SQS trigger Lambda worker
handler.add_event_source(SqsEventSource(sqs_queue, batch_size=1))
# Option1: Create S3 Bucket, all new objects in this bucket will be transmitted by Lambda Worker
s3bucket = s3.Bucket(self, "s3_new_migrate")
s3bucket.grant_read(handler)
s3bucket.add_event_notification(s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(sqs_queue))
# Option2: Allow Exist S3 Buckets to be read by Lambda functions.
# Lambda Jobsender will scan and compare the these buckets and trigger Lambda Workers to transmit
bucket_name = ''
for b in bucket_para:
if bucket_name != b['src_bucket']: # 如果列了多个相同的Bucket，就跳过
bucket_name = b['src_bucket']
s3exist_bucket = s3.Bucket.from_bucket_name(
self,
bucket_name, # 用这个做id
bucket_name=bucket_name)
s3exist_bucket.grant_read(handler_jobsender)
s3exist_bucket.grant_read(handler)
# Allow Lambda read ssm parameters
ssm_bucket_para.grant_read(handler_jobsender)
ssm_credential_para.grant_read(handler)
ssm_credential_para.grant_read(handler_jobsender)
ssm_parameter_ignore_list.grant_read(handler_jobsender)
# Schedule cron event to trigger Lambda Jobsender per hour:
event.Rule(self,
'cron_trigger_jobsender',
schedule=event.Schedule.rate(core.Duration.hours(1)),
targets=[target.LambdaFunction(handler_jobsender)])
# Create Lambda logs filter to create network traffic metric
handler.log_group.add_metric_filter(
"Complete-bytes",
metric_name="Complete-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Complete", bytes, key]'))
handler.log_group.add_metric_filter(
"Uploading-bytes",
metric_name="Uploading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Uploading", bytes, key]'))
handler.log_group.add_metric_filter(
"Downloading-bytes",
metric_name="Downloading-bytes",
metric_namespace="s3_migrate",
metric_value="$bytes",
filter_pattern=logs.FilterPattern.literal(
'[info, date, sn, p="--->Downloading", bytes, key]'))
lambda_metric_Complete = cw.Metric(namespace="s3_migrate",
metric_name="Complete-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Upload = cw.Metric(namespace="s3_migrate",
metric_name="Uploading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
lambda_metric_Download = cw.Metric(namespace="s3_migrate",
metric_name="Downloading-bytes",
statistic="Sum",
period=core.Duration.minutes(1))
handler.log_group.add_metric_filter(
"ERROR",
metric_name="ERROR-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"ERROR"'))
handler.log_group.add_metric_filter(
"WARNING",
metric_name="WARNING-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"WARNING"'))
# Task timed out
handler.log_group.add_metric_filter(
"TIMEOUT",
metric_name="TIMEOUT-Logs",
metric_namespace="s3_migrate",
metric_value="1",
filter_pattern=logs.FilterPattern.literal('"Task timed out"'))
log_metric_ERROR = cw.Metric(namespace="s3_migrate",
metric_name="ERROR-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
log_metric_WARNING = cw.Metric(namespace="s3_migrate",
metric_name="WARNING-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
log_metric_TIMEOUT = cw.Metric(namespace="s3_migrate",
metric_name="TIMEOUT-Logs",
statistic="Sum",
period=core.Duration.minutes(1))
# Dashboard to monitor SQS and Lambda
board = cw.Dashboard(self, "s3_migrate_serverless")
board.add_widgets(
cw.GraphWidget(title="Lambda-NETWORK",
left=[
lambda_metric_Download, lambda_metric_Upload,
lambda_metric_Complete
]),
# TODO: here monitor all lambda concurrency not just the working one. Limitation from CDK
# Lambda now supports monitor single lambda concurrency, will change this after CDK support
cw.GraphWidget(title="Lambda-all-concurrent",
left=[
handler.metric_all_concurrent_executions(
period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="Lambda-invocations/errors/throttles",
left=[
handler.metric_invocations(
period=core.Duration.minutes(1)),
handler.metric_errors(period=core.Duration.minutes(1)),
handler.metric_throttles(period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="Lambda-duration",
left=[
handler.metric_duration(period=core.Duration.minutes(1))
]),
)
board.add_widgets(
cw.GraphWidget(
title="SQS-Jobs",
left=[
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1))
]),
cw.GraphWidget(
title="SQS-DeadLetterQueue",
left=[
sqs_queue_DLQ.
metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1))
]),
cw.GraphWidget(title="ERROR/WARNING Logs",
left=[log_metric_ERROR],
right=[log_metric_WARNING, log_metric_TIMEOUT]),
cw.SingleValueWidget(
title="Running/Waiting and Dead Jobs",
metrics=[
sqs_queue.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)),
sqs_queue.metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_not_visible(
period=core.Duration.minutes(1)),
sqs_queue_DLQ.
metric_approximate_number_of_messages_visible(
period=core.Duration.minutes(1))
],
height=6))
# Alarm for queue - DLQ
alarm_DLQ = cw.Alarm(
self,
"SQS_DLQ",
metric=sqs_queue_DLQ.metric_approximate_number_of_messages_visible(
),
threshold=0,
comparison_operator=cw.ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluation_periods=1,
datapoints_to_alarm=1)
alarm_topic = sns.Topic(self, "SQS queue-DLQ has dead letter")
alarm_topic.add_subscription(
subscription=sub.EmailSubscription(alarm_email))
alarm_DLQ.add_alarm_action(action.SnsAction(alarm_topic))
core.CfnOutput(self,
"Dashboard",
value="CloudWatch Dashboard name s3_migrate_serverless")
def __init__(self, scope: core.Construct, id: str, emr_launch_stack,
artifact_bucket, output_bucket, **kwargs):
super().__init__(scope, id, **kwargs)
launch_function = emr_launch_stack.launch_function
# Create DynamoDB table for tracking
dynamo_table = dynamo.Table(
self,
"dynamotable",
partition_key=dynamo.Attribute(name="BatchId",
type=dynamo.AttributeType.STRING),
sort_key=dynamo.Attribute(name="Name",
type=dynamo.AttributeType.STRING),
billing_mode=dynamo.BillingMode.PAY_PER_REQUEST)
emr_role = aws_iam.Role.from_role_arn(
self, "emr_role_iam", role_arn=emr_launch_stack.instance_role_arn)
emr_role.add_to_policy(
aws_iam.PolicyStatement(actions=["dynamodb:*"],
resources=[dynamo_table.table_arn]))
emr_role.add_to_policy(
aws_iam.PolicyStatement(actions=[
"logs:CreateLogStream", "logs:DescribeLogStreams",
"logs:CreateLogGroup", "logs:PutLogEvents", "ec2:DescribeTags"
],
resources=["*"]))
# SNS Topics for Success/Failures messages from our Pipeline
self.success_topic = sns.Topic(self, 'SuccessTopic')
self.failure_topic = sns.Topic(self, 'FailureTopic')
# Upload artifacts to S3
step_code = s3d.BucketDeployment(
self,
id='sparkscript',
destination_bucket=artifact_bucket,
destination_key_prefix='steps',
sources=[
s3d.Source.asset('infrastructure/emr_orchestration/steps/')
])
# Create a Chain to receive Failure messages
fail = emr_chains.Fail(self,
'FailChain',
message=sfn.TaskInput.from_data_at('$.Error'),
subject='Pipeline Failure',
topic=self.failure_topic)
# # Define a Task to Terminate the Cluster on failure
terminate_failed_cluster = emr_tasks.TerminateClusterBuilder.build(
self,
'TerminateFailedCluster',
name='Terminate Failed Cluster',
cluster_id=sfn.TaskInput.from_data_at(
'$.LaunchClusterResult.ClusterId').value,
result_path='$.TerminateResult').add_catch(fail,
errors=['States.ALL'],
result_path='$.Error')
terminate_failed_cluster.next(fail)
# Use a NestedStateMachine to launch the cluster
launch_cluster = emr_chains.NestedStateMachine(
self,
'NestedStateMachine',
name='Launch Cluster StateMachine',
state_machine=launch_function.state_machine,
fail_chain=fail)
pyspark_step = emr_chains.AddStepWithArgumentOverrides(
self,
'PySparkDataIngestion',
emr_step=emr_code.EMRStep(
name=f'Data Ingestion - PySpark Job',
jar='command-runner.jar',
args=[
'spark-submit', '--master', 'yarn', '--deploy-mode',
'client', '--packages',
'com.audienceproject:spark-dynamodb_2.12:1.1.2',
os.path.join(f's3://{artifact_bucket.bucket_name}',
'steps', 'data_ingestion.py'), '--batch-id',
'DynamoDB.BatchId', '--batch-metadata-table-name',
dynamo_table.table_name, '--output-bucket',
output_bucket.bucket_name, '--region',
os.environ["CDK_DEFAULT_REGION"]
]),
cluster_id=sfn.TaskInput.from_data_at(
'$.LaunchClusterResult.ClusterId').value,
result_path='$.PySparkResult',
fail_chain=terminate_failed_cluster)
pyspark_example_step = emr_chains.AddStepWithArgumentOverrides(
self,
'PySparkDataPreparation',
emr_step=emr_code.EMRStep(
name=f'Data Preparation - PySpark Job',
jar='command-runner.jar',
args=[
'spark-submit', '--master', 'yarn', '--deploy-mode',
'client',
os.path.join(f's3://{artifact_bucket.bucket_name}',
'steps', 'data_preparation.py'), '--batch-id',
'DynamoDB.BatchId', '--batch-metadata-table-name',
dynamo_table.table_name, '--input-bucket',
output_bucket.bucket_name, '--region',
os.environ["CDK_DEFAULT_REGION"]
]),
cluster_id=sfn.TaskInput.from_data_at(
'$.LaunchClusterResult.ClusterId').value,
result_path='$.PySparkResult',
fail_chain=terminate_failed_cluster)
# Define a Task to Terminate the Cluster
terminate_cluster = emr_tasks.TerminateClusterBuilder.build(
self,
'TerminateCluster',
name='Terminate Cluster',
cluster_id=sfn.TaskInput.from_data_at(
'$.LaunchClusterResult.ClusterId').value,
result_path='$.TerminateResult').add_catch(fail,
errors=['States.ALL'],
result_path='$.Error')
# A Chain for Success notification when the pipeline completes
success = emr_chains.Success(
self,
'SuccessChain',
message=sfn.TaskInput.from_data_at('$.TerminateResult'),
subject='Pipeline Succeeded',
topic=self.success_topic)
# Assemble the Pipeline
definition = sfn.Chain \
.start(launch_cluster) \
.next(pyspark_step) \
.next(pyspark_example_step) \
.next(terminate_cluster) \
.next(success)
# Create the State Machine
self.state_machine = sfn.StateMachine(
self,
'PySparkExampleStateMachine',
state_machine_name='pyspark-example-pipeline',
definition=definition)
self.dynamo_table = dynamo_table
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
vpc = aws_ec2.Vpc(
self,
"OctemberVPC",
max_azs=2,
# subnet_configuration=[{
# "cidrMask": 24,
# "name": "Public",
# "subnetType": aws_ec2.SubnetType.PUBLIC,
# },
# {
# "cidrMask": 24,
# "name": "Private",
# "subnetType": aws_ec2.SubnetType.PRIVATE
# },
# {
# "cidrMask": 28,
# "name": "Isolated",
# "subnetType": aws_ec2.SubnetType.ISOLATED,
# "reserved": True
# }
# ],
gateway_endpoints={
"S3":
aws_ec2.GatewayVpcEndpointOptions(
service=aws_ec2.GatewayVpcEndpointAwsService.S3)
})
dynamo_db_endpoint = vpc.add_gateway_endpoint(
"DynamoDbEndpoint",
service=aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB)
s3_bucket = s3.Bucket(
self,
"s3bucket",
bucket_name="octember-bizcard-{region}-{account}".format(
region=core.Aws.REGION, account=core.Aws.ACCOUNT_ID))
api = apigw.RestApi(
self,
"BizcardImageUploader",
rest_api_name="BizcardImageUploader",
description="This service serves uploading bizcard images into s3.",
endpoint_types=[apigw.EndpointType.REGIONAL],
binary_media_types=["image/png", "image/jpg"],
deploy=True,
deploy_options=apigw.StageOptions(stage_name="v1"))
rest_api_role = aws_iam.Role(
self,
"ApiGatewayRoleForS3",
role_name="ApiGatewayRoleForS3FullAccess",
assumed_by=aws_iam.ServicePrincipal("apigateway.amazonaws.com"),
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess")
])
list_objects_responses = [
apigw.IntegrationResponse(
status_code="200",
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationResponse.html#aws_cdk.aws_apigateway.IntegrationResponse.response_parameters
# The response parameters from the backend response that API Gateway sends to the method response.
# Use the destination as the key and the source as the value:
# - The destination must be an existing response parameter in the MethodResponse property.
# - The source must be an existing method request parameter or a static value.
response_parameters={
'method.response.header.Timestamp':
'integration.response.header.Date',
'method.response.header.Content-Length':
'integration.response.header.Content-Length',
'method.response.header.Content-Type':
'integration.response.header.Content-Type'
}),
apigw.IntegrationResponse(status_code="400",
selection_pattern="4\d{2}"),
apigw.IntegrationResponse(status_code="500",
selection_pattern="5\d{2}")
]
list_objects_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=list_objects_responses)
get_s3_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="GET",
path='/',
options=list_objects_integration_options)
api.root.add_method(
"GET",
get_s3_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Timestamp': False,
'method.response.header.Content-Length': False,
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={'method.request.header.Content-Type': False})
get_s3_folder_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=list_objects_responses,
#XXX: https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_apigateway/IntegrationOptions.html#aws_cdk.aws_apigateway.IntegrationOptions.request_parameters
# Specify request parameters as key-value pairs (string-to-string mappings), with a destination as the key and a source as the value.
# The source must be an existing method request parameter or a static value.
request_parameters={
"integration.request.path.bucket": "method.request.path.folder"
})
get_s3_folder_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="GET",
path="{bucket}",
options=get_s3_folder_integration_options)
s3_folder = api.root.add_resource('{folder}')
s3_folder.add_method(
"GET",
get_s3_folder_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Timestamp': False,
'method.response.header.Content-Length': False,
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.folder': True
})
get_s3_item_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=list_objects_responses,
request_parameters={
"integration.request.path.bucket":
"method.request.path.folder",
"integration.request.path.object": "method.request.path.item"
})
get_s3_item_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="GET",
path="{bucket}/{object}",
options=get_s3_item_integration_options)
s3_item = s3_folder.add_resource('{item}')
s3_item.add_method(
"GET",
get_s3_item_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Timestamp': False,
'method.response.header.Content-Length': False,
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.folder': True,
'method.request.path.item': True
})
put_s3_item_integration_options = apigw.IntegrationOptions(
credentials_role=rest_api_role,
integration_responses=[
apigw.IntegrationResponse(status_code="200"),
apigw.IntegrationResponse(status_code="400",
selection_pattern="4\d{2}"),
apigw.IntegrationResponse(status_code="500",
selection_pattern="5\d{2}")
],
request_parameters={
"integration.request.header.Content-Type":
"method.request.header.Content-Type",
"integration.request.path.bucket":
"method.request.path.folder",
"integration.request.path.object": "method.request.path.item"
})
put_s3_item_integration = apigw.AwsIntegration(
service="s3",
integration_http_method="PUT",
path="{bucket}/{object}",
options=put_s3_item_integration_options)
s3_item.add_method(
"PUT",
put_s3_item_integration,
authorization_type=apigw.AuthorizationType.IAM,
api_key_required=False,
method_responses=[
apigw.MethodResponse(
status_code="200",
response_parameters={
'method.response.header.Content-Type': False
},
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.folder': True,
'method.request.path.item': True
})
ddb_table = dynamodb.Table(
self,
"BizcardImageMetaInfoDdbTable",
table_name="OctemberBizcardImgMeta",
partition_key=dynamodb.Attribute(
name="image_id", type=dynamodb.AttributeType.STRING),
billing_mode=dynamodb.BillingMode.PROVISIONED,
read_capacity=15,
write_capacity=5)
img_kinesis_stream = kinesis.Stream(
self, "BizcardImagePath", stream_name="octember-bizcard-image")
# create lambda function
trigger_textract_lambda_fn = _lambda.Function(
self,
"TriggerTextExtractorFromImage",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="TriggerTextExtractorFromImage",
handler="trigger_text_extract_from_s3_image.lambda_handler",
description="Trigger to extract text from an image in S3",
code=_lambda.Code.asset(
"./src/main/python/TriggerTextExtractFromS3Image"),
environment={
'REGION_NAME': core.Aws.REGION,
'DDB_TABLE_NAME': ddb_table.table_name,
'KINESIS_STREAM_NAME': img_kinesis_stream.stream_name
},
timeout=core.Duration.minutes(5))
ddb_table_rw_policy_statement = aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
resources=[ddb_table.table_arn],
actions=[
"dynamodb:BatchGetItem", "dynamodb:Describe*",
"dynamodb:List*", "dynamodb:GetItem", "dynamodb:Query",
"dynamodb:Scan", "dynamodb:BatchWriteItem",
"dynamodb:DeleteItem", "dynamodb:PutItem",
"dynamodb:UpdateItem", "dax:Describe*", "dax:List*",
"dax:GetItem", "dax:BatchGetItem", "dax:Query", "dax:Scan",
"dax:BatchWriteItem", "dax:DeleteItem", "dax:PutItem",
"dax:UpdateItem"
])
trigger_textract_lambda_fn.add_to_role_policy(
ddb_table_rw_policy_statement)
trigger_textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=[img_kinesis_stream.stream_arn],
actions=[
"kinesis:Get*", "kinesis:List*",
"kinesis:Describe*",
"kinesis:PutRecord",
"kinesis:PutRecords"
]))
# assign notification for the s3 event type (ex: OBJECT_CREATED)
s3_event_filter = s3.NotificationKeyFilter(prefix="bizcard-raw-img/",
suffix=".jpg")
s3_event_source = S3EventSource(s3_bucket,
events=[s3.EventType.OBJECT_CREATED],
filters=[s3_event_filter])
trigger_textract_lambda_fn.add_event_source(s3_event_source)
#XXX: https://github.com/aws/aws-cdk/issues/2240
# To avoid to create extra Lambda Functions with names like LogRetentionaae0aa3c5b4d4f87b02d85b201efdd8a
# if log_retention=aws_logs.RetentionDays.THREE_DAYS is added to the constructor props
log_group = aws_logs.LogGroup(
self,
"TriggerTextractLogGroup",
log_group_name="/aws/lambda/TriggerTextExtractorFromImage",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(trigger_textract_lambda_fn)
text_kinesis_stream = kinesis.Stream(
self, "BizcardTextData", stream_name="octember-bizcard-txt")
textract_lambda_fn = _lambda.Function(
self,
"GetTextFromImage",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="GetTextFromImage",
handler="get_text_from_s3_image.lambda_handler",
description="extract text from an image in S3",
code=_lambda.Code.asset("./src/main/python/GetTextFromS3Image"),
environment={
'REGION_NAME': core.Aws.REGION,
'DDB_TABLE_NAME': ddb_table.table_name,
'KINESIS_STREAM_NAME': text_kinesis_stream.stream_name
},
timeout=core.Duration.minutes(5))
textract_lambda_fn.add_to_role_policy(ddb_table_rw_policy_statement)
textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=[text_kinesis_stream.stream_arn],
actions=[
"kinesis:Get*", "kinesis:List*",
"kinesis:Describe*",
"kinesis:PutRecord",
"kinesis:PutRecords"
]))
textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
s3_bucket.bucket_arn, "{}/*".format(
s3_bucket.bucket_arn)
],
"actions": [
"s3:AbortMultipartUpload", "s3:GetBucketLocation",
"s3:GetObject", "s3:ListBucket",
"s3:ListBucketMultipartUploads", "s3:PutObject"
]
}))
textract_lambda_fn.add_to_role_policy(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["*"],
actions=["textract:*"]))
img_kinesis_event_source = KinesisEventSource(
img_kinesis_stream,
batch_size=100,
starting_position=_lambda.StartingPosition.LATEST)
textract_lambda_fn.add_event_source(img_kinesis_event_source)
log_group = aws_logs.LogGroup(
self,
"GetTextFromImageLogGroup",
log_group_name="/aws/lambda/GetTextFromImage",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(textract_lambda_fn)
sg_use_bizcard_es = aws_ec2.SecurityGroup(
self,
"BizcardSearchClientSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard elasticsearch client',
security_group_name='use-octember-bizcard-es')
core.Tag.add(sg_use_bizcard_es, 'Name', 'use-octember-bizcard-es')
sg_bizcard_es = aws_ec2.SecurityGroup(
self,
"BizcardSearchSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for octember bizcard elasticsearch',
security_group_name='octember-bizcard-es')
core.Tag.add(sg_bizcard_es, 'Name', 'octember-bizcard-es')
sg_bizcard_es.add_ingress_rule(peer=sg_bizcard_es,
connection=aws_ec2.Port.all_tcp(),
description='octember-bizcard-es')
sg_bizcard_es.add_ingress_rule(peer=sg_use_bizcard_es,
connection=aws_ec2.Port.all_tcp(),
description='use-octember-bizcard-es')
sg_ssh_access = aws_ec2.SecurityGroup(
self,
"BastionHostSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for bastion host',
security_group_name='octember-bastion-host-sg')
core.Tag.add(sg_ssh_access, 'Name', 'octember-bastion-host')
sg_ssh_access.add_ingress_rule(peer=aws_ec2.Peer.any_ipv4(),
connection=aws_ec2.Port.tcp(22),
description='ssh access')
bastion_host = aws_ec2.BastionHostLinux(
self,
"BastionHost",
vpc=vpc,
instance_type=aws_ec2.InstanceType('t3.nano'),
security_group=sg_ssh_access,
subnet_selection=aws_ec2.SubnetSelection(
subnet_type=aws_ec2.SubnetType.PUBLIC))
bastion_host.instance.add_security_group(sg_use_bizcard_es)
#XXX: aws cdk elastsearch example - https://github.com/aws/aws-cdk/issues/2873
es_cfn_domain = aws_elasticsearch.CfnDomain(
self,
'BizcardSearch',
elasticsearch_cluster_config={
"dedicatedMasterCount": 3,
"dedicatedMasterEnabled": True,
"dedicatedMasterType": "t2.medium.elasticsearch",
"instanceCount": 2,
"instanceType": "t2.medium.elasticsearch",
"zoneAwarenessEnabled": True
},
ebs_options={
"ebsEnabled": True,
"volumeSize": 10,
"volumeType": "gp2"
},
domain_name="octember-bizcard",
elasticsearch_version="7.1",
encryption_at_rest_options={"enabled": False},
access_policies={
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Principal": {
"AWS": "*"
},
"Action":
["es:Describe*", "es:List*", "es:Get*", "es:ESHttp*"],
"Resource":
self.format_arn(service="es",
resource="domain",
resource_name="octember-bizcard/*")
}]
},
snapshot_options={"automatedSnapshotStartHour": 17},
vpc_options={
"securityGroupIds": [sg_bizcard_es.security_group_id],
"subnetIds":
vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids
})
core.Tag.add(es_cfn_domain, 'Name', 'octember-bizcard-es')
s3_lib_bucket_name = self.node.try_get_context("lib_bucket_name")
#XXX: https://github.com/aws/aws-cdk/issues/1342
s3_lib_bucket = s3.Bucket.from_bucket_name(self, id,
s3_lib_bucket_name)
es_lib_layer = _lambda.LayerVersion(
self,
"ESLib",
layer_version_name="es-lib",
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
code=_lambda.Code.from_bucket(s3_lib_bucket,
"var/octember-es-lib.zip"))
redis_lib_layer = _lambda.LayerVersion(
self,
"RedisLib",
layer_version_name="redis-lib",
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
code=_lambda.Code.from_bucket(s3_lib_bucket,
"var/octember-redis-lib.zip"))
#XXX: Deploy lambda in VPC - https://github.com/aws/aws-cdk/issues/1342
upsert_to_es_lambda_fn = _lambda.Function(
self,
"UpsertBizcardToES",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="UpsertBizcardToElasticSearch",
handler="upsert_bizcard_to_es.lambda_handler",
description="Upsert bizcard text into elasticsearch",
code=_lambda.Code.asset("./src/main/python/UpsertBizcardToES"),
environment={
'ES_HOST': es_cfn_domain.attr_domain_endpoint,
'ES_INDEX': 'octember_bizcard',
'ES_TYPE': 'bizcard'
},
timeout=core.Duration.minutes(5),
layers=[es_lib_layer],
security_groups=[sg_use_bizcard_es],
vpc=vpc)
text_kinesis_event_source = KinesisEventSource(
text_kinesis_stream,
batch_size=99,
starting_position=_lambda.StartingPosition.LATEST)
upsert_to_es_lambda_fn.add_event_source(text_kinesis_event_source)
log_group = aws_logs.LogGroup(
self,
"UpsertBizcardToESLogGroup",
log_group_name="/aws/lambda/UpsertBizcardToElasticSearch",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(upsert_to_es_lambda_fn)
firehose_role_policy_doc = aws_iam.PolicyDocument()
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
s3_bucket.bucket_arn, "{}/*".format(
s3_bucket.bucket_arn)
],
"actions": [
"s3:AbortMultipartUpload", "s3:GetBucketLocation",
"s3:GetObject", "s3:ListBucket",
"s3:ListBucketMultipartUploads", "s3:PutObject"
]
}))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=["*"],
actions=[
"glue:GetTable",
"glue:GetTableVersion",
"glue:GetTableVersions"
]))
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(effect=aws_iam.Effect.ALLOW,
resources=[text_kinesis_stream.stream_arn],
actions=[
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords"
]))
firehose_log_group_name = "/aws/kinesisfirehose/octember-bizcard-txt-to-s3"
firehose_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
effect=aws_iam.Effect.ALLOW,
#XXX: The ARN will be formatted as follows:
# arn:{partition}:{service}:{region}:{account}:{resource}{sep}}{resource-name}
resources=[
self.format_arn(service="logs",
resource="log-group",
resource_name="{}:log-stream:*".format(
firehose_log_group_name),
sep=":")
],
actions=["logs:PutLogEvents"]))
firehose_role = aws_iam.Role(
self,
"FirehoseDeliveryRole",
role_name="FirehoseDeliveryRole",
assumed_by=aws_iam.ServicePrincipal("firehose.amazonaws.com"),
#XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221
inline_policies={"firehose_role_policy": firehose_role_policy_doc})
bizcard_text_to_s3_delivery_stream = aws_kinesisfirehose.CfnDeliveryStream(
self,
"BizcardTextToS3",
delivery_stream_name="octember-bizcard-txt-to-s3",
delivery_stream_type="KinesisStreamAsSource",
kinesis_stream_source_configuration={
"kinesisStreamArn": text_kinesis_stream.stream_arn,
"roleArn": firehose_role.role_arn
},
extended_s3_destination_configuration={
"bucketArn": s3_bucket.bucket_arn,
"bufferingHints": {
"intervalInSeconds": 60,
"sizeInMBs": 1
},
"cloudWatchLoggingOptions": {
"enabled": True,
"logGroupName": firehose_log_group_name,
"logStreamName": "S3Delivery"
},
"compressionFormat": "GZIP",
"prefix": "bizcard-text/",
"roleArn": firehose_role.role_arn
})
sg_use_bizcard_es_cache = aws_ec2.SecurityGroup(
self,
"BizcardSearchCacheClientSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard search query cache client',
security_group_name='use-octember-bizcard-es-cache')
core.Tag.add(sg_use_bizcard_es_cache, 'Name',
'use-octember-bizcard-es-cache')
sg_bizcard_es_cache = aws_ec2.SecurityGroup(
self,
"BizcardSearchCacheSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard search query cache',
security_group_name='octember-bizcard-es-cache')
core.Tag.add(sg_bizcard_es_cache, 'Name', 'octember-bizcard-es-cache')
sg_bizcard_es_cache.add_ingress_rule(
peer=sg_use_bizcard_es_cache,
connection=aws_ec2.Port.tcp(6379),
description='use-octember-bizcard-es-cache')
es_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup(
self,
"QueryCacheSubnetGroup",
description="subnet group for octember-bizcard-es-cache",
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids,
cache_subnet_group_name='octember-bizcard-es-cache')
es_query_cache = aws_elasticache.CfnCacheCluster(
self,
"BizcardSearchQueryCache",
cache_node_type="cache.t3.small",
num_cache_nodes=1,
engine="redis",
engine_version="5.0.5",
auto_minor_version_upgrade=False,
cluster_name="octember-bizcard-es-cache",
snapshot_retention_limit=3,
snapshot_window="17:00-19:00",
preferred_maintenance_window="mon:19:00-mon:20:30",
#XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098
#cache_subnet_group_name=es_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC
cache_subnet_group_name='octember-bizcard-es-cache',
vpc_security_group_ids=[sg_bizcard_es_cache.security_group_id])
#XXX: If you're going to launch your cluster in an Amazon VPC, you need to create a subnet group before you start creating a cluster.
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticache-cache-cluster.html#cfn-elasticache-cachecluster-cachesubnetgroupname
es_query_cache.add_depends_on(es_query_cache_subnet_group)
#XXX: add more than 2 security groups
# https://github.com/aws/aws-cdk/blob/ea10f0d141a48819ec0000cd7905feda993870a9/packages/%40aws-cdk/aws-lambda/lib/function.ts#L387
# https://github.com/aws/aws-cdk/issues/1555
# https://github.com/aws/aws-cdk/pull/5049
bizcard_search_lambda_fn = _lambda.Function(
self,
"BizcardSearchServer",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="BizcardSearchProxy",
handler="es_search_bizcard.lambda_handler",
description="Proxy server to search bizcard text",
code=_lambda.Code.asset("./src/main/python/SearchBizcard"),
environment={
'ES_HOST': es_cfn_domain.attr_domain_endpoint,
'ES_INDEX': 'octember_bizcard',
'ES_TYPE': 'bizcard',
'ELASTICACHE_HOST': es_query_cache.attr_redis_endpoint_address
},
timeout=core.Duration.minutes(1),
layers=[es_lib_layer, redis_lib_layer],
security_groups=[sg_use_bizcard_es, sg_use_bizcard_es_cache],
vpc=vpc)
#XXX: create API Gateway + LambdaProxy
search_api = apigw.LambdaRestApi(
self,
"BizcardSearchAPI",
handler=bizcard_search_lambda_fn,
proxy=False,
rest_api_name="BizcardSearch",
description="This service serves searching bizcard text.",
endpoint_types=[apigw.EndpointType.REGIONAL],
deploy=True,
deploy_options=apigw.StageOptions(stage_name="v1"))
bizcard_search = search_api.root.add_resource('search')
bizcard_search.add_method(
"GET",
method_responses=[
apigw.MethodResponse(
status_code="200",
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
])
sg_use_bizcard_graph_db = aws_ec2.SecurityGroup(
self,
"BizcardGraphDbClientSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for octember bizcard graph db client',
security_group_name='use-octember-bizcard-neptune')
core.Tag.add(sg_use_bizcard_graph_db, 'Name',
'use-octember-bizcard-neptune')
sg_bizcard_graph_db = aws_ec2.SecurityGroup(
self,
"BizcardGraphDbSG",
vpc=vpc,
allow_all_outbound=True,
description='security group for octember bizcard graph db',
security_group_name='octember-bizcard-neptune')
core.Tag.add(sg_bizcard_graph_db, 'Name', 'octember-bizcard-neptune')
sg_bizcard_graph_db.add_ingress_rule(
peer=sg_bizcard_graph_db,
connection=aws_ec2.Port.tcp(8182),
description='octember-bizcard-neptune')
sg_bizcard_graph_db.add_ingress_rule(
peer=sg_use_bizcard_graph_db,
connection=aws_ec2.Port.tcp(8182),
description='use-octember-bizcard-neptune')
bizcard_graph_db_subnet_group = aws_neptune.CfnDBSubnetGroup(
self,
"NeptuneSubnetGroup",
db_subnet_group_description=
"subnet group for octember-bizcard-neptune",
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids,
db_subnet_group_name='octember-bizcard-neptune')
bizcard_graph_db = aws_neptune.CfnDBCluster(
self,
"BizcardGraphDB",
availability_zones=vpc.availability_zones,
db_subnet_group_name=bizcard_graph_db_subnet_group.
db_subnet_group_name,
db_cluster_identifier="octember-bizcard",
backup_retention_period=1,
preferred_backup_window="08:45-09:15",
preferred_maintenance_window="sun:18:00-sun:18:30",
vpc_security_group_ids=[sg_bizcard_graph_db.security_group_id])
bizcard_graph_db.add_depends_on(bizcard_graph_db_subnet_group)
bizcard_graph_db_instance = aws_neptune.CfnDBInstance(
self,
"BizcardGraphDBInstance",
db_instance_class="db.r5.large",
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
availability_zone=vpc.availability_zones[0],
db_cluster_identifier=bizcard_graph_db.db_cluster_identifier,
db_instance_identifier="octember-bizcard",
preferred_maintenance_window="sun:18:00-sun:18:30")
bizcard_graph_db_instance.add_depends_on(bizcard_graph_db)
bizcard_graph_db_replica_instance = aws_neptune.CfnDBInstance(
self,
"BizcardGraphDBReplicaInstance",
db_instance_class="db.r5.large",
allow_major_version_upgrade=False,
auto_minor_version_upgrade=False,
availability_zone=vpc.availability_zones[-1],
db_cluster_identifier=bizcard_graph_db.db_cluster_identifier,
db_instance_identifier="octember-bizcard-replica",
preferred_maintenance_window="sun:18:00-sun:18:30")
bizcard_graph_db_replica_instance.add_depends_on(bizcard_graph_db)
bizcard_graph_db_replica_instance.add_depends_on(
bizcard_graph_db_instance)
gremlinpython_lib_layer = _lambda.LayerVersion(
self,
"GremlinPythonLib",
layer_version_name="gremlinpython-lib",
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
code=_lambda.Code.from_bucket(
s3_lib_bucket, "var/octember-gremlinpython-lib.zip"))
#XXX: https://github.com/aws/aws-cdk/issues/1342
upsert_to_neptune_lambda_fn = _lambda.Function(
self,
"UpsertBizcardToGraphDB",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="UpsertBizcardToNeptune",
handler="upsert_bizcard_to_graph_db.lambda_handler",
description="Upsert bizcard into neptune",
code=_lambda.Code.asset(
"./src/main/python/UpsertBizcardToGraphDB"),
environment={
'REGION_NAME': core.Aws.REGION,
'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_endpoint,
'NEPTUNE_PORT': bizcard_graph_db.attr_port
},
timeout=core.Duration.minutes(5),
layers=[gremlinpython_lib_layer],
security_groups=[sg_use_bizcard_graph_db],
vpc=vpc)
upsert_to_neptune_lambda_fn.add_event_source(text_kinesis_event_source)
log_group = aws_logs.LogGroup(
self,
"UpsertBizcardToGraphDBLogGroup",
log_group_name="/aws/lambda/UpsertBizcardToNeptune",
retention=aws_logs.RetentionDays.THREE_DAYS)
log_group.grant_write(upsert_to_neptune_lambda_fn)
sg_use_bizcard_neptune_cache = aws_ec2.SecurityGroup(
self,
"BizcardNeptuneCacheClientSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard recommendation query cache client',
security_group_name='use-octember-bizcard-neptune-cache')
core.Tag.add(sg_use_bizcard_neptune_cache, 'Name',
'use-octember-bizcard-es-cache')
sg_bizcard_neptune_cache = aws_ec2.SecurityGroup(
self,
"BizcardNeptuneCacheSG",
vpc=vpc,
allow_all_outbound=True,
description=
'security group for octember bizcard recommendation query cache',
security_group_name='octember-bizcard-neptune-cache')
core.Tag.add(sg_bizcard_neptune_cache, 'Name',
'octember-bizcard-neptune-cache')
sg_bizcard_neptune_cache.add_ingress_rule(
peer=sg_use_bizcard_neptune_cache,
connection=aws_ec2.Port.tcp(6379),
description='use-octember-bizcard-neptune-cache')
recomm_query_cache_subnet_group = aws_elasticache.CfnSubnetGroup(
self,
"RecommQueryCacheSubnetGroup",
description="subnet group for octember-bizcard-neptune-cache",
subnet_ids=vpc.select_subnets(
subnet_type=aws_ec2.SubnetType.PRIVATE).subnet_ids,
cache_subnet_group_name='octember-bizcard-neptune-cache')
recomm_query_cache = aws_elasticache.CfnCacheCluster(
self,
"BizcardRecommQueryCache",
cache_node_type="cache.t3.small",
num_cache_nodes=1,
engine="redis",
engine_version="5.0.5",
auto_minor_version_upgrade=False,
cluster_name="octember-bizcard-neptune-cache",
snapshot_retention_limit=3,
snapshot_window="17:00-19:00",
preferred_maintenance_window="mon:19:00-mon:20:30",
#XXX: Do not use referece for "cache_subnet_group_name" - https://github.com/aws/aws-cdk/issues/3098
#cache_subnet_group_name=recomm_query_cache_subnet_group.cache_subnet_group_name, # Redis cluster goes to wrong VPC
cache_subnet_group_name='octember-bizcard-neptune-cache',
vpc_security_group_ids=[
sg_bizcard_neptune_cache.security_group_id
])
recomm_query_cache.add_depends_on(recomm_query_cache_subnet_group)
bizcard_recomm_lambda_fn = _lambda.Function(
self,
"BizcardRecommender",
runtime=_lambda.Runtime.PYTHON_3_7,
function_name="BizcardRecommender",
handler="neptune_recommend_bizcard.lambda_handler",
description="This service serves PYMK(People You May Know).",
code=_lambda.Code.asset("./src/main/python/RecommendBizcard"),
environment={
'REGION_NAME': core.Aws.REGION,
'NEPTUNE_ENDPOINT': bizcard_graph_db.attr_read_endpoint,
'NEPTUNE_PORT': bizcard_graph_db.attr_port,
'ELASTICACHE_HOST':
recomm_query_cache.attr_redis_endpoint_address
},
timeout=core.Duration.minutes(1),
layers=[gremlinpython_lib_layer, redis_lib_layer],
security_groups=[
sg_use_bizcard_graph_db, sg_use_bizcard_neptune_cache
],
vpc=vpc)
#XXX: create API Gateway + LambdaProxy
recomm_api = apigw.LambdaRestApi(
self,
"BizcardRecommendAPI",
handler=bizcard_recomm_lambda_fn,
proxy=False,
rest_api_name="BizcardRecommend",
description="This service serves PYMK(People You May Know).",
endpoint_types=[apigw.EndpointType.REGIONAL],
deploy=True,
deploy_options=apigw.StageOptions(stage_name="v1"))
bizcard_recomm = recomm_api.root.add_resource('pymk')
bizcard_recomm.add_method(
"GET",
method_responses=[
apigw.MethodResponse(
status_code="200",
response_models={'application/json': apigw.EmptyModel()}),
apigw.MethodResponse(status_code="400"),
apigw.MethodResponse(status_code="500")
])
sagemaker_notebook_role_policy_doc = aws_iam.PolicyDocument()
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
"arn:aws:s3:::aws-neptune-notebook",
"arn:aws:s3:::aws-neptune-notebook/*"
],
"actions": ["s3:GetObject", "s3:ListBucket"]
}))
sagemaker_notebook_role_policy_doc.add_statements(
aws_iam.PolicyStatement(
**{
"effect":
aws_iam.Effect.ALLOW,
"resources": [
"arn:aws:neptune-db:{region}:{account}:{cluster_id}/*".
format(region=core.Aws.REGION,
account=core.Aws.ACCOUNT_ID,
cluster_id=bizcard_graph_db.
attr_cluster_resource_id)
],
"actions": ["neptune-db:connect"]
}))
sagemaker_notebook_role = aws_iam.Role(
self,
'SageMakerNotebookForNeptuneWorkbenchRole',
role_name='AWSNeptuneNotebookRole-OctemberBizcard',
assumed_by=aws_iam.ServicePrincipal('sagemaker.amazonaws.com'),
#XXX: use inline_policies to work around https://github.com/aws/aws-cdk/issues/5221
inline_policies={
'AWSNeptuneNotebook': sagemaker_notebook_role_policy_doc
})
neptune_wb_lifecycle_content = '''#!/bin/bash
sudo -u ec2-user -i <<'EOF'
echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc
echo "export GRAPH_NOTEBOOK_HOST={NeptuneClusterEndpoint}" >> ~/.bashrc
echo "export GRAPH_NOTEBOOK_PORT={NeptuneClusterPort}" >> ~/.bashrc
echo "export NEPTUNE_LOAD_FROM_S3_ROLE_ARN=''" >> ~/.bashrc
echo "export AWS_REGION={AWS_Region}" >> ~/.bashrc
aws s3 cp s3://aws-neptune-notebook/graph_notebook.tar.gz /tmp/graph_notebook.tar.gz
rm -rf /tmp/graph_notebook
tar -zxvf /tmp/graph_notebook.tar.gz -C /tmp
/tmp/graph_notebook/install.sh
EOF
'''.format(NeptuneClusterEndpoint=bizcard_graph_db.attr_endpoint,
NeptuneClusterPort=bizcard_graph_db.attr_port,
AWS_Region=core.Aws.REGION)
neptune_wb_lifecycle_config_prop = aws_sagemaker.CfnNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleHookProperty(
content=core.Fn.base64(neptune_wb_lifecycle_content))
neptune_wb_lifecycle_config = aws_sagemaker.CfnNotebookInstanceLifecycleConfig(
self,
'NpetuneWorkbenchLifeCycleConfig',
notebook_instance_lifecycle_config_name=
'AWSNeptuneWorkbenchOctemberBizcardLCConfig',
on_start=[neptune_wb_lifecycle_config_prop])
neptune_workbench = aws_sagemaker.CfnNotebookInstance(
self,
'NeptuneWorkbench',
instance_type='ml.t2.medium',
role_arn=sagemaker_notebook_role.role_arn,
lifecycle_config_name=neptune_wb_lifecycle_config.
notebook_instance_lifecycle_config_name,
notebook_instance_name='OctemberBizcard-NeptuneWorkbench',
root_access='Disabled',
security_group_ids=[sg_use_bizcard_graph_db.security_group_name],
subnet_id=bizcard_graph_db_subnet_group.subnet_ids[0])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# If left unchecked this pattern could "fan out" on the transform and load
# lambdas to the point that it consumes all resources on the account. This is
# why we are limiting concurrency to 2 on all 3 lambdas. Feel free to raise this.
lambda_throttle_size = 2
####
# DynamoDB Table
# This is where our transformed data ends up
####
table = dynamo_db.Table(self,
"TransformedData",
partition_key=dynamo_db.Attribute(
name="id",
type=dynamo_db.AttributeType.STRING))
####
# S3 Landing Bucket
# This is where the user uploads the file to be transformed
####
bucket = s3.Bucket(self, "LandingBucket")
####
# Queue that listens for S3 Bucket events
####
queue = sqs.Queue(self,
'newObjectInLandingBucketEventQueue',
visibility_timeout=core.Duration.seconds(300))
bucket.add_event_notification(s3.EventType.OBJECT_CREATED,
s3n.SqsDestination(queue))
# EventBridge Permissions
event_bridge_put_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=['*'],
actions=['events:PutEvents'])
####
# Fargate ECS Task Creation to pull data from S3
#
# Fargate is used here because if you had a seriously large file,
# you could stream the data to fargate for as long as needed before
# putting the data onto eventbridge or up the memory/storage to
# download the whole file. Lambda has limitations on runtime and
# memory/storage
####
vpc = ec2.Vpc(self, "Vpc", max_azs=2)
logging = ecs.AwsLogDriver(stream_prefix='TheEventBridgeETL',
log_retention=logs.RetentionDays.ONE_WEEK)
cluster = ecs.Cluster(self, 'Ec2Cluster', vpc=vpc)
task_definition = ecs.TaskDefinition(
self,
'FargateTaskDefinition',
memory_mib="512",
cpu="256",
compatibility=ecs.Compatibility.FARGATE)
# We need to give our fargate container permission to put events on our EventBridge
task_definition.add_to_task_role_policy(event_bridge_put_policy)
# Grant fargate container access to the object that was uploaded to s3
bucket.grant_read(task_definition.task_role)
container = task_definition.add_container(
'AppContainer',
image=ecs.ContainerImage.from_asset(
'container/s3DataExtractionTask'),
logging=logging,
environment={
'S3_BUCKET_NAME': bucket.bucket_name,
'S3_OBJECT_KEY': ''
})
####
# Lambdas
#
# These are used for 4 phases:
#
# Extract - kicks of ecs fargate task to download data and splinter to eventbridge events
# Transform - takes the two comma separated strings and produces a json object
# Load - inserts the data into dynamodb
# Observe - This is a lambda that subscribes to all events and logs them centrally
####
subnet_ids = []
for subnet in vpc.private_subnets:
subnet_ids.append(subnet.subnet_id)
####
# Extract
# defines an AWS Lambda resource to trigger our fargate ecs task
####
extract_lambda = _lambda.Function(
self,
"extractLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="s3SqsEventConsumer.handler",
code=_lambda.Code.from_asset("lambda_fns/extract"),
reserved_concurrent_executions=lambda_throttle_size,
environment={
"CLUSTER_NAME": cluster.cluster_name,
"TASK_DEFINITION": task_definition.task_definition_arn,
"SUBNETS": json.dumps(subnet_ids),
"CONTAINER_NAME": container.container_name
})
queue.grant_consume_messages(extract_lambda)
extract_lambda.add_event_source(_event.SqsEventSource(queue=queue))
extract_lambda.add_to_role_policy(event_bridge_put_policy)
run_task_policy_statement = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=[task_definition.task_definition_arn],
actions=['ecs:RunTask'])
extract_lambda.add_to_role_policy(run_task_policy_statement)
task_execution_role_policy_statement = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=[
task_definition.obtain_execution_role().role_arn,
task_definition.task_role.role_arn
],
actions=['iam:PassRole'])
extract_lambda.add_to_role_policy(task_execution_role_policy_statement)
####
# Transform
# defines a lambda to transform the data that was extracted from s3
####
transform_lambda = _lambda.Function(
self,
"TransformLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="transform.handler",
code=_lambda.Code.from_asset("lambda_fns/transform"),
reserved_concurrent_executions=lambda_throttle_size,
timeout=core.Duration.seconds(3))
transform_lambda.add_to_role_policy(event_bridge_put_policy)
# Create EventBridge rule to route extraction events
transform_rule = events.Rule(
self,
'transformRule',
description='Data extracted from S3, Needs transformed',
event_pattern=events.EventPattern(
source=['cdkpatterns.the-eventbridge-etl'],
detail_type=['s3RecordExtraction'],
detail={"status": ["extracted"]}))
transform_rule.add_target(
targets.LambdaFunction(handler=transform_lambda))
####
# Load
# load the transformed data in dynamodb
####
load_lambda = _lambda.Function(
self,
"LoadLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="load.handler",
code=_lambda.Code.from_asset("lambda_fns/load"),
reserved_concurrent_executions=lambda_throttle_size,
timeout=core.Duration.seconds(3),
environment={"TABLE_NAME": table.table_name})
load_lambda.add_to_role_policy(event_bridge_put_policy)
table.grant_read_write_data(load_lambda)
load_rule = events.Rule(
self,
'loadRule',
description='Data transformed, Needs loaded into dynamodb',
event_pattern=events.EventPattern(
source=['cdkpatterns.the-eventbridge-etl'],
detail_type=['transform'],
detail={"status": ["transformed"]}))
load_rule.add_target(targets.LambdaFunction(handler=load_lambda))
####
# Observe
# Watch for all cdkpatterns.the-eventbridge-etl events and log them centrally
####
observe_lambda = _lambda.Function(
self,
"ObserveLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="observe.handler",
code=_lambda.Code.from_asset("lambda_fns/observe"),
reserved_concurrent_executions=lambda_throttle_size,
timeout=core.Duration.seconds(3))
observe_rule = events.Rule(
self,
'observeRule',
description='all events are caught here and logged centrally',
event_pattern=events.EventPattern(
source=['cdkpatterns.the-eventbridge-etl']))
observe_rule.add_target(targets.LambdaFunction(handler=observe_lambda))
