def __init__(self, scope: core.Construct, id: str,
api_gateway: aws_apigateway.RestApi, **kwargs):
super().__init__(scope, id, **kwargs)
# create dynamo table
job_table = aws_dynamodb.Table(
self,
"demo_table",
partition_key=aws_dynamodb.Attribute(
name="job_id", type=aws_dynamodb.AttributeType.STRING))
# Create the Queue
sqs_queue = aws_sqs.Queue(self, "SQSQueue")
# Create the Handler
lambda_sqs_role = aws_iam.Role(
self,
id='lambda-sqs-role',
assumed_by=aws_iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
aws_iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaSQSQueueExecutionRole')
])
ecr_image = aws_lambda.EcrImageCode.from_asset_image(
directory=os.path.join(os.getcwd(),
"startuptoolbag/app/lambda-sqs-handler"))
lambda_handler_function = aws_lambda.Function(
self,
id="lambdaSQSHandlerFunction",
description="Handles/Valdates background requests and puts on SQS",
code=ecr_image,
handler=aws_lambda.Handler.FROM_IMAGE,
runtime=aws_lambda.Runtime.FROM_IMAGE,
environment={'SQS_NAME': sqs_queue.queue_name},
role=lambda_sqs_role,
allow_public_subnet=True,
memory_size=128,
reserved_concurrent_executions=10,
timeout=core.Duration.seconds(10))
sqs_queue.grant_send_messages(lambda_handler_function)
# Create the Background Worker
ecr_image = aws_lambda.EcrImageCode.from_asset_image(
directory=os.path.join(os.getcwd(),
"startuptoolbag/app/lambda-sqs-bworker"))
background_function = aws_lambda.Function(
self,
id="lambdaSQSDrivenBackgroundWorker",
description="Pulls from SQS and is a background worker",
code=ecr_image,
handler=aws_lambda.Handler.FROM_IMAGE,
runtime=aws_lambda.Runtime.FROM_IMAGE,
environment={
'SQS_NAME': sqs_queue.queue_name,
'TABLE_NAME': job_table.table_name
},
role=lambda_sqs_role,
allow_public_subnet=True,
memory_size=128,
reserved_concurrent_executions=10,
timeout=core.Duration.seconds(10))
background_function.add_event_source(
eventsources.SqsEventSource(sqs_queue))
background_function.add_environment("TABLE_NAME", job_table.table_name)
job_table.grant_write_data(background_function)
# Create the Lambda Serving Requests
ecr_image = aws_lambda.EcrImageCode.from_asset_image(
directory=os.path.join(
os.getcwd(), "startuptoolbag/app/lambda-dynamodb-server"))
reader_function = aws_lambda.Function(
self,
id="lambdaDynamo Server",
description="Handles API Requests backed by dynamo",
code=ecr_image,
handler=aws_lambda.Handler.FROM_IMAGE,
runtime=aws_lambda.Runtime.FROM_IMAGE,
environment={'TABLE_NAME': job_table.table_name},
role=lambda_sqs_role,
allow_public_subnet=True,
memory_size=128,
reserved_concurrent_executions=10,
timeout=core.Duration.seconds(10))
job_table.grant_read_data(reader_function)
foo_r = api_gateway.root.add_resource("jobs")
foo_r.add_method('GET',
aws_apigateway.LambdaIntegration(reader_function))
foo_r.add_method(
'POST', aws_apigateway.LambdaIntegration(lambda_handler_function))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
mybucket = s3.Bucket(
self,
ID + '-bucket',
encryption=s3.BucketEncryption.S3_MANAGED,
versioned=True,
lifecycle_rules=[
s3.LifecycleRule(
enabled=True,
noncurrent_version_expiration=core.Duration.days(30),
)
])
cloudtrail.Trail(
self,
ID + '-trail',
enable_file_validation=True,
include_global_service_events=True,
is_multi_region_trail=True,
send_to_cloud_watch_logs=True,
cloud_watch_logs_retention=logs.RetentionDays.ONE_WEEK,
)
func_diff_notice = lambda_.Function(
self,
ID + '-func_diff_notice',
code=lambda_.Code.asset('./functions/artifacts/'),
handler='app.diff_notice',
runtime=lambda_.Runtime.PYTHON_3_7,
log_retention=logs.RetentionDays.ONE_MONTH,
memory_size=128,
timeout=core.Duration.seconds(60),
tracing=lambda_.Tracing.ACTIVE)
func_diff_notice.add_to_role_policy(
iam.PolicyStatement(
actions=['ssm:GetParameter'],
resources=['*'],
))
func_codebuild_alert = lambda_.Function(
self,
ID + '-func_codebuild_alert',
code=lambda_.Code.asset('./functions/artifacts/'),
handler='app.codebuild_alert',
runtime=lambda_.Runtime.PYTHON_3_7,
log_retention=logs.RetentionDays.ONE_MONTH,
memory_size=128,
timeout=core.Duration.seconds(60),
tracing=lambda_.Tracing.ACTIVE)
func_codebuild_alert.add_to_role_policy(
iam.PolicyStatement(
actions=['ssm:GetParameter'],
resources=['*'],
))
codebuild_project = codebuild.Project(
self,
ID + '-codebuild-project',
environment=codebuild.BuildEnvironment(
build_image=codebuild.LinuxBuildImage.STANDARD_2_0,
compute_type=codebuild.ComputeType.SMALL),
build_spec=codebuild.BuildSpec.from_object({
'version': 0.2,
'phases': {
'install': {
'runtime-versions': {
'ruby': 2.6
},
'commands': ['gem install roadworker']
},
'build': {
'commands': [
'roadwork --export --target-zone ' +
HOSTED_ZONE_NAME + ' --output Routefile',
'aws s3 cp Routefile s3://' +
mybucket.bucket_name + '/Routefile'
]
}
}
}))
codebuild_project.add_to_role_policy(
iam.PolicyStatement(
actions=['s3:putObject'],
resources=[mybucket.bucket_arn + '/Routefile'],
))
codebuild_project.add_to_role_policy(
iam.PolicyStatement(
actions=['route53:List*', 'route53:Get*'],
resources=['*'],
))
codebuild_project.on_build_failed(
ID + '-rule-on_build_failed',
target=targets.LambdaFunction(func_codebuild_alert))
rule = events.Rule(self, ID + '-rule', enabled=True)
rule.add_event_pattern(
source=['aws.route53'],
detail_type=['AWS API Call via CloudTrail'],
detail={
'eventSource': ['route53.amazonaws.com'],
'eventName': ['ChangeResourceRecordSets'],
'requestParameters': {
'hostedZoneId': [HOSTED_ZONE_ID]
},
},
)
rule.add_target(targets.LambdaFunction(func_diff_notice))
rule.add_target(targets.CodeBuildProject(codebuild_project))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Setting up a salck bot Lambda function
bot_handler = aws_lambda.Function(
self,
"HitterBot",
runtime=aws_lambda.Runtime.GO_1_X,
handler="main",
timeout=core.Duration.seconds(900),
memory_size=2048,
code=aws_lambda.AssetCode(path="./lambda"))
# Creating Mutex Table in DynamoDB
mutex_table = aws_dynamodb.Table(
self,
"HitterMutexTable",
partition_key=aws_dynamodb.Attribute(
name="ID", type=aws_dynamodb.AttributeType.STRING),
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
time_to_live_attribute="TTL",
removal_policy=core.RemovalPolicy.DESTROY,
)
# Creating URL Table in DynamoDB
url_table = aws_dynamodb.Table(
self,
"HitterURLTable",
partition_key=aws_dynamodb.Attribute(
name="ID", type=aws_dynamodb.AttributeType.STRING),
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
time_to_live_attribute="TTL",
removal_policy=core.RemovalPolicy.DESTROY,
)
# Creating a bucket to be used in a Pre-Signed URL
bucket = aws_s3.Bucket(
self,
"HitterS3",
removal_policy=core.RemovalPolicy.RETAIN,
block_public_access=aws_s3.BlockPublicAccess.BLOCK_ALL,
)
# Configuring the S3 Lifecycle
bucket.add_lifecycle_rule(expiration=core.Duration.days(2))
# Setting permission to the salck bot Lambda function
mutex_table.grant_read_write_data(bot_handler)
url_table.grant_read_write_data(bot_handler)
bucket.grant_put(bot_handler)
bucket.grant_read(bot_handler)
bot_handler.add_to_role_policy(
aws_iam.PolicyStatement(
resources=["*"],
actions=[
"comprehend:BatchDetectDominantLanguage",
"translate:TranslateText"
]))
# Setting environment variables to the salck bot Lambda function
bot_handler.add_environment('SLACK_OAUTH_ACCESS_TOKEN',
SLACK_OAUTH_ACCESS_TOKEN)
bot_handler.add_environment('SLACK_VERIFICATION_TOKEN',
SLACK_VERIFICATION_TOKEN)
bot_handler.add_environment('MUTEX_TABLE_NAME', mutex_table.table_name)
bot_handler.add_environment('URL_TABLE_NAME', url_table.table_name)
bot_handler.add_environment('S3_BUCKET_NAME', bucket.bucket_name)
bot_handler.add_environment('DEBUG_LOG', "false")
# Creating an API Gateway for a slack bot
bot_api = aws_apigateway.LambdaRestApi(self,
"HitterBotAPI",
handler=bot_handler)
# Only once for the hosted zone.
hosted_zone = aws_route53.HostedZone.from_hosted_zone_attributes(
self,
'HitterHostedZone',
hosted_zone_id=ZONE_ID,
zone_name=ZONE_NAME)
# Set the domain for a bot
bot_subdomain = "hitter"
bot_domain_name = bot_subdomain + '.' + ZONE_NAME
# Using AWS ACM to create a certificate for a bot
bot_cert = aws_certificatemanager.DnsValidatedCertificate(
self,
'HitterBotCertificate',
domain_name=bot_domain_name,
hosted_zone=hosted_zone)
# Add the domain name to the api and the A record to our hosted zone for a bot
bot_domain = bot_api.add_domain_name('HitterBotDomain',
certificate=bot_cert,
domain_name=bot_domain_name)
# Set the A record for a bot
aws_route53.ARecord(
self,
'HitterBotARecord',
record_name=bot_subdomain,
zone=hosted_zone,
target=aws_route53.RecordTarget.from_alias(
aws_route53_targets.ApiGatewayDomain(bot_domain)))
# Setting up Short URL Lambda function
short_url_handler = aws_lambda.Function(
self,
"HitterShortURL",
runtime=aws_lambda.Runtime.GO_1_X,
handler="main",
timeout=core.Duration.seconds(900),
memory_size=128,
code=aws_lambda.AssetCode(path="./lambda_api"))
# Setting environment variables to the Short URL Lambda function
url_table.grant_read_data(short_url_handler)
# Setting environment variables to the salck bot Lambda function
short_url_handler.add_environment('URL_TABLE_NAME',
url_table.table_name)
short_url_handler.add_environment('DEBUG_LOG', "false")
# Creating an API Gateway for a Short URL
short_url_api = aws_apigateway.LambdaRestApi(self,
"HitterShortURLAPI",
handler=short_url_handler)
# Set the domain for a Short URL
short_url_subdomain = "sl"
short_url_domain_name = short_url_subdomain + '.' + ZONE_NAME
short_url = 'https://' + short_url_domain_name
# Using AWS ACM to create a certificate for a Short URL
short_url_cert = aws_certificatemanager.DnsValidatedCertificate(
self,
'HitterShortURLCertificate',
domain_name=short_url_domain_name,
hosted_zone=hosted_zone)
# Add the domain name to the api and the A record to our hosted zone for a Short URL
short_url_domain = short_url_api.add_domain_name(
'HitterShortURLDomain',
certificate=short_url_cert,
domain_name=short_url_domain_name)
# Set the A record for a Short URL
aws_route53.ARecord(
self,
'HitterShortURLARecord',
record_name=short_url_subdomain,
zone=hosted_zone,
target=aws_route53.RecordTarget.from_alias(
aws_route53_targets.ApiGatewayDomain(short_url_domain)))
# Set the URL of the API Gateway to receive the shortened URL to an environment variable.
bot_handler.add_environment('API_BASE_URL', short_url)
def __init__(self, scope: cdk.Stack, id: str, **kwargs):
super().__init__(scope, id, **kwargs)
self.output_bucket = aws_s3.Bucket(
self,
"BucketTwitterStreamOutput",
bucket_name=self.stack_name,
)
self.bucket_url = self.output_bucket.bucket_regional_domain_name
# Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own
self.iam_role = aws_iam.Role(
self,
"IAMRoleTwitterStreamKinesisFHToS3",
role_name="KinesisFirehoseToS3-{}".format(self.stack_name),
assumed_by=aws_iam.ServicePrincipal(
service='firehose.amazonaws.com'),
)
# S3 bucket actions
self.s3_iam_policy_statement = aws_iam.PolicyStatement()
actions = [
"s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket",
"s3:ListBucketMultipartUploads", "s3:PutObject"
]
for action in actions:
self.s3_iam_policy_statement.add_action(action)
self.s3_iam_policy_statement.add_resource(
self.output_bucket.bucket_arn)
self.s3_iam_policy_statement.add_resource(
self.output_bucket.bucket_arn + "/*")
# CW error log setup
self.s3_error_logs_group = aws_logs.LogGroup(
self,
"S3ErrorLogsGroup",
log_group_name="{}-s3-errors".format(self.stack_name))
self.s3_error_logs_stream = aws_logs.LogStream(
self,
"S3ErrorLogsStream",
log_group=self.s3_error_logs_group,
log_stream_name='s3Backup')
self.firehose = aws_kinesisfirehose.CfnDeliveryStream(
self,
"FirehoseTwitterStream",
delivery_stream_name="{}-raw".format(self.stack_name),
delivery_stream_type="DirectPut",
s3_destination_configuration={
'bucketArn': self.output_bucket.bucket_arn,
'bufferingHints': {
'intervalInSeconds': 120,
'sizeInMBs': 10
},
'compressionFormat': 'UNCOMPRESSED',
'roleArn': self.iam_role.role_arn,
'cloudWatchLoggingOptions': {
'enabled': True,
'logGroupName': "{}-raw".format(self.stack_name),
'logStreamName': 's3BackupRaw'
},
'prefix': 'twitter-raw/'
},
)
# TODO: Only attach what's needed for this policy, right now i'm lazy and attaching all policies
self.iam_policy = aws_iam.Policy(
self,
"IAMPolicyTwitterStreamKinesisFHToS3",
policy_name="KinesisFirehoseToS3-{}".format(self.stack_name),
statements=[self.s3_iam_policy_statement],
)
self.iam_policy.attach_to_role(self.iam_role)
# Because kinesis firehose bindings are to direct CF, we have to create IAM policy/role and attach on our own
self.curator_firehose = aws_kinesisfirehose.CfnDeliveryStream(
self,
"CuratorFirehoseStream",
delivery_stream_name="{}-curator".format(self.stack_name),
delivery_stream_type="DirectPut",
s3_destination_configuration={
'bucketArn': self.output_bucket.bucket_arn,
'bufferingHints': {
'intervalInSeconds': 120,
'sizeInMBs': 10
},
'compressionFormat': 'UNCOMPRESSED',
'roleArn': self.iam_role.role_arn,
'cloudWatchLoggingOptions': {
'enabled': True,
'logGroupName': "{}-curator".format(self.stack_name),
'logStreamName': 's3BackupCurator'
},
'prefix': 'twitter-curated/'
},
)
def zip_package():
cwd = os.getcwd()
file_name = 'curator-lambda.zip'
zip_file = cwd + '/' + file_name
os.chdir('src/')
sh.zip('-r9', zip_file, '.')
os.chdir(cwd)
return file_name, zip_file
_, zip_file = zip_package()
self.twitter_stream_curator_lambda_function = aws_lambda.Function(
self,
"TwitterStreamCuratorLambdaFunction",
function_name="{}-curator".format(self.stack_name),
code=aws_lambda.AssetCode(zip_file),
handler="sentiment_analysis.lambda_handler",
runtime=aws_lambda.Runtime.PYTHON37,
tracing=aws_lambda.Tracing.Active,
description=
"Triggers from S3 PUT event for twitter stream data and transorms it to clean json syntax with sentiment analysis attached",
environment={
"STACK_NAME": self.stack_name,
"FIREHOSE_STREAM": self.curator_firehose.delivery_stream_name
},
memory_size=128,
timeout=120,
log_retention_days=aws_logs.RetentionDays.OneWeek,
)
# Permission to talk to comprehend for sentiment analysis
self.comprehend_iam_policy_statement = aws_iam.PolicyStatement()
self.comprehend_iam_policy_statement.add_action('comprehend:*')
self.comprehend_iam_policy_statement.add_all_resources()
self.twitter_stream_curator_lambda_function.add_to_role_policy(
self.comprehend_iam_policy_statement)
# Permission to put in kinesis firehose
self.curator_firehose_iam_policy_statement = aws_iam.PolicyStatement()
self.curator_firehose_iam_policy_statement.add_action('firehose:Put*')
self.curator_firehose_iam_policy_statement.add_resource(
self.curator_firehose.delivery_stream_arn)
self.twitter_stream_curator_lambda_function.add_to_role_policy(
self.curator_firehose_iam_policy_statement)
# Attaching the policy to the IAM role for KFH
self.output_bucket.grant_read(
self.twitter_stream_curator_lambda_function)
self.twitter_stream_curator_lambda_function.add_event_source(
aws_lambda_event_sources.S3EventSource(
bucket=self.output_bucket,
events=[aws_s3.EventType.ObjectCreated],
filters=[aws_s3.NotificationKeyFilter(prefix="twitter-raw/")]))
def __init__(self, scope: core.Construct, id: str,
resource: aws_apigateway.Resource):
super().__init__(scope, id)
# DynamoDB
self.table = aws_dynamodb.Table(
self,
"stock_table",
partition_key=aws_dynamodb.Attribute(
name="id", type=aws_dynamodb.AttributeType.STRING),
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
table_name="stock_table",
removal_policy=core.RemovalPolicy.DESTROY)
# Lambdas
create_lambda = aws_lambda.Function(
self,
"stock_create_lambda",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="lambda_function.lambda_handler",
code=aws_lambda.Code.asset("./services/stock/create"),
memory_size=256,
timeout=core.Duration.seconds(10),
function_name="stock_create_lambda")
create_lambda.add_environment("STOCK_TABLE", self.table.table_name)
self.table.grant_read_write_data(create_lambda)
self.find_lambda = aws_lambda.Function(
self,
"stock_find_lambda",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="lambda_function.lambda_handler",
code=aws_lambda.Code.asset("./services/stock/find"),
memory_size=256,
timeout=core.Duration.seconds(10),
function_name="stock_find_lambda")
self.find_lambda.add_environment("STOCK_TABLE", self.table.table_name)
self.table.grant_read_write_data(self.find_lambda)
self.add_lambda = aws_lambda.Function(
self,
"stock_add_lambda",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="lambda_function.lambda_handler",
code=aws_lambda.Code.asset("./services/stock/add"),
memory_size=256,
timeout=core.Duration.seconds(10),
function_name="stock_add_lambda")
self.add_lambda.add_environment("STOCK_TABLE", self.table.table_name)
self.table.grant_read_write_data(self.add_lambda)
self.subtract_lambda = aws_lambda.Function(
self,
"stock_subtract_lambda",
runtime=aws_lambda.Runtime.PYTHON_3_6,
handler="lambda_function.lambda_handler",
code=aws_lambda.Code.asset("./services/stock/subtract"),
memory_size=256,
timeout=core.Duration.seconds(10),
function_name="stock_subtract_lambda")
self.subtract_lambda.add_environment("STOCK_TABLE",
self.table.table_name)
self.table.grant_read_write_data(self.subtract_lambda)
# API Gateway integration
create_integration = aws_apigateway.LambdaIntegration(create_lambda)
find_integration = aws_apigateway.LambdaIntegration(self.find_lambda)
add_integration = aws_apigateway.LambdaIntegration(self.add_lambda)
subtract_integration = aws_apigateway.LambdaIntegration(
self.subtract_lambda)
# API Gateway
# POST /stock/item/create/{price}
create = resource.add_resource("item").add_resource(
"create").add_resource("{price}")
create.add_method("POST", create_integration)
# GET /stock/find/{item_id}
find = resource.add_resource("find").add_resource("{item_id}")
find.add_method("GET", find_integration)
# POST /stock/add/{item_id}/{number}
add = resource.add_resource("add").add_resource(
"{item_id}").add_resource("{number}")
add.add_method("POST", add_integration)
# POST /stock/subtract/{item_id}/{number}
subtract = resource.add_resource("subtract").add_resource(
"{item_id}").add_resource("{number}")
subtract.add_method("POST", subtract_integration)
def __init__(self, scope: core.Construct, id: str, stack_prefix: str,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Model all required resources
ddb_table = _ddb.Table(
self,
id='{}-data'.format(stack_prefix),
table_name='{}-data'.format(stack_prefix),
partition_key=_ddb.Attribute(name='ID',
type=_ddb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.
DESTROY, # THIS IS NOT RECOMMENDED FOR PRODUCTION USE
read_capacity=1,
write_capacity=1)
## IAM Roles
lambda_role = _iam.Role(
self,
id='{}-lambda-role'.format(stack_prefix),
assumed_by=_iam.ServicePrincipal('lambda.amazonaws.com'))
cw_policy_statement = _iam.PolicyStatement(effect=_iam.Effect.ALLOW)
cw_policy_statement.add_actions("logs:CreateLogGroup")
cw_policy_statement.add_actions("logs:CreateLogStream")
cw_policy_statement.add_actions("logs:PutLogEvents")
cw_policy_statement.add_actions("logs:DescribeLogStreams")
cw_policy_statement.add_resources("*")
lambda_role.add_to_policy(cw_policy_statement)
# Add role for DynamoDB
dynamodb_policy_statement = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW)
dynamodb_policy_statement.add_actions("dynamodb:PutItem")
dynamodb_policy_statement.add_actions("dynamodb:GetItem")
dynamodb_policy_statement.add_actions("dynamodb:Scan")
dynamodb_policy_statement.add_actions("dynamodb:Query")
dynamodb_policy_statement.add_actions("dynamodb:ConditionCheckItem")
dynamodb_policy_statement.add_resources(ddb_table.table_arn)
lambda_role.add_to_policy(dynamodb_policy_statement)
## AWS Lambda Functions
fnLambda_storeData = _lambda.Function(
self,
"{}-function-storeData".format(stack_prefix),
code=_lambda.AssetCode("../lambda-functions/store-data"),
handler="app.handler",
timeout=core.Duration.seconds(60),
role=lambda_role,
runtime=_lambda.Runtime.PYTHON_3_8)
fnLambda_storeData.add_environment("TABLE_NAME", ddb_table.table_name)
fnLambda_listData = _lambda.Function(
self,
"{}-function-getData".format(stack_prefix),
code=_lambda.AssetCode("../lambda-functions/list-data"),
handler="app.handler",
role=lambda_role,
timeout=core.Duration.seconds(60),
runtime=_lambda.Runtime.PYTHON_3_8)
fnLambda_listData.add_environment("TABLE_NAME", ddb_table.table_name)
core.CfnOutput(self,
"{}-output-dynamodbTable".format(stack_prefix),
value=ddb_table.table_name,
export_name="{}-ddbTable".format(stack_prefix))
core.CfnOutput(
self,
"{}-output-lambdaStoreData".format(stack_prefix),
value=fnLambda_storeData.function_name,
export_name="{}-lambdaStoreDataName".format(stack_prefix))
core.CfnOutput(
self,
"{}-output-lambdaListData".format(stack_prefix),
value=fnLambda_listData.function_name,
export_name="{}-lambdaListDataName".format(stack_prefix))
def __init__(self, scope: core.Construct, id: str, lambda_role: iam.Role) -> None:
super().__init__(scope, id)
self.emails_bucket = s3.Bucket(
self,
f'{get_stack_name()}EmailsBucket',
removal_policy=core.RemovalPolicy.DESTROY,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
)
self.emails_bucket.add_to_resource_policy(
iam.PolicyStatement(actions=['s3:PutObject'],
resources=[f'arn:aws:s3:::{self.emails_bucket.bucket_name}/*'],
principals=[iam.ServicePrincipal('ses.amazonaws.com')],
conditions={ "StringEquals": { "aws:Referer": core.Stack.of(self).account } }
)
)
bucket_name_output = core.CfnOutput(self, id="EmailBucket", value=self.emails_bucket.bucket_name)
bucket_name_output.override_logical_id("EmailBucket")
admin_submit_lambda = aws_lambda.Function(
self,
'AdminDataSubmit',
runtime=aws_lambda.Runtime.PYTHON_3_8,
code=aws_lambda.Code.from_asset(self._LAMBDA_ASSET_DIR),
handler='functions.handler.admin_submit',
role=lambda_role,
environment={
EMAIL_BUCKET_ENV_VAR: self.emails_bucket.bucket_name
},
)
teacher_submit_lambda = aws_lambda.Function(
self,
'TeacherDataSubmit',
runtime=aws_lambda.Runtime.PYTHON_3_8,
code=aws_lambda.Code.from_asset(self._LAMBDA_ASSET_DIR),
handler='functions.handler.teacher_submit',
role=lambda_role,
environment={
EMAIL_BUCKET_ENV_VAR: self.emails_bucket.bucket_name
},
)
self.emails_bucket.grant_read_write(admin_submit_lambda)
self.emails_bucket.grant_read_write(teacher_submit_lambda)
ruleset_name = ses.ReceiptRuleSet(scope=self, id="DataSubmissionReceiptRuleSet",
receipt_rule_set_name=f'{get_stack_name()}RecieptRules',
rules=[
ses.ReceiptRuleOptions(
receipt_rule_name=f'{get_stack_name()}AdminSubmitRule',
recipients=[f'adminsubmit@{KESHER_DOMAIN_NAME}'],
actions=[
ses_actions.S3(bucket=self.emails_bucket, object_key_prefix="AdminIncomingEmail"),
ses_actions.Lambda(function=admin_submit_lambda)
],
),
ses.ReceiptRuleOptions(
receipt_rule_name=f'{get_stack_name()}TeacherSubmitRule',
recipients=[f'teachersubmit@{KESHER_DOMAIN_NAME}'],
actions=[
ses_actions.S3(bucket=self.emails_bucket, object_key_prefix="TeacherIncomingEmail"),
ses_actions.Lambda(function=teacher_submit_lambda),
]
)
]
)
ruleset_name_output = core.CfnOutput(self, id="RuleSetName", value=ruleset_name.receipt_rule_set_name)
ruleset_name_output.override_logical_id("RuleSetName")
def __init__(
self,
scope: core.Construct,
id_: str,
vpc_stack,
elastic_stack,
update_lambda_zip=False,
**kwargs,
) -> None:
super().__init__(scope, id_, **kwargs)
# if update lambda zip (including if zip doesn't exist)
if (
update_lambda_zip
or not pathlib.Path(os.path.join(dirname, "kibana_lambda.zip")).exists()
):
# rebuild the lambda if changed
call(["docker", "build", "--tag", "kibana-lambda", "."], cwd=dirname)
call(
["docker", "create", "-ti", "--name", "dummy", "kibana-lambda", "bash"],
cwd=dirname,
)
call(["docker", "cp", "dummy:/tmp/kibana_lambda.zip", "."], cwd=dirname)
call(["docker", "rm", "-f", "dummy"], cwd=dirname)
kibana_bucket = s3.Bucket(
self,
"kibana_bucket",
public_read_access=False,
block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
removal_policy=core.RemovalPolicy.DESTROY,
)
# tag the bucket
core.Tags.of(kibana_bucket).add("project", constants["PROJECT_TAG"])
# the lambda behind the api
kibana_lambda = lambda_.Function(
self,
"kibana_lambda",
description="kibana api gateway lambda",
code=lambda_.Code.from_asset(os.path.join(dirname, "kibana_lambda.zip")),
handler="lambda_function.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_8,
vpc=vpc_stack.get_vpc,
security_groups=[elastic_stack.elastic_security_group],
log_retention=logs.RetentionDays.ONE_WEEK,
)
# tag the lambda
core.Tags.of(kibana_lambda).add("project", constants["PROJECT_TAG"])
# create policies for the lambda
kibana_lambda_policy = iam.PolicyStatement(
effect=iam.Effect.ALLOW, actions=["s3:*"], resources=["*"],
)
# add the role permissions
kibana_lambda.add_to_role_policy(statement=kibana_lambda_policy)
# the api gateway
kibana_api = apigw.LambdaRestApi(
self, "kibana_api", handler=kibana_lambda, binary_media_types=["*/*"]
)
# tag the api gateway
core.Tags.of(kibana_api).add("project", constants["PROJECT_TAG"])
kibana_identity = cloudfront.OriginAccessIdentity(self, "kibana_identity")
kibana_api_domain = "/".join(kibana_api.url.split("/")[1:-2])[1:]
kibana_api_path = f'/{"/".join(kibana_api.url.split("/")[-2:])}'
# create the cloudfront distribution
kibana_distribution = cloudfront.CloudFrontWebDistribution(
self,
"kibana_distribution",
origin_configs=[
# the lambda source for kibana
cloudfront.SourceConfiguration(
custom_origin_source=cloudfront.CustomOriginConfig(
domain_name=kibana_api_domain,
origin_protocol_policy=cloudfront.OriginProtocolPolicy.HTTPS_ONLY,
),
origin_path="/prod",
behaviors=[
cloudfront.Behavior(
is_default_behavior=True,
allowed_methods=cloudfront.CloudFrontAllowedMethods.ALL,
cached_methods=cloudfront.CloudFrontAllowedCachedMethods.GET_HEAD_OPTIONS,
compress=False,
forwarded_values=CfnDistribution.ForwardedValuesProperty(
query_string=True,
cookies=CfnDistribution.CookiesProperty(forward="all"),
headers=[
"Content-Type",
"Accept",
"Accept-Encoding",
"kbn-name",
"kbn-version",
],
),
)
],
),
# the s3 bucket source for kibana
cloudfront.SourceConfiguration(
s3_origin_source=cloudfront.S3OriginConfig(
s3_bucket_source=kibana_bucket,
origin_access_identity=kibana_identity,
),
behaviors=[
cloudfront.Behavior(
is_default_behavior=False,
path_pattern="bucket_cached/*",
allowed_methods=cloudfront.CloudFrontAllowedMethods.GET_HEAD,
cached_methods=cloudfront.CloudFrontAllowedCachedMethods.GET_HEAD,
compress=True,
)
],
),
],
)
# tag the cloudfront distribution
core.Tags.of(kibana_distribution).add("project", constants["PROJECT_TAG"])
# needs api and bucket to be available
kibana_distribution.node.add_dependency(kibana_api)
# kibana bucket empty policies
kibana_bucket_empty_policy = [
iam.PolicyStatement(
effect=iam.Effect.ALLOW, actions=["s3:ListBucket"], resources=["*"],
),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=["s3:DeleteObject"],
resources=[f"{kibana_bucket.bucket_arn}/*"],
),
]
# create the custom resource
kibana_bucket_empty = CustomResource(
self,
"kibana_bucket_empty",
PhysicalId="kibanaBucketEmpty",
Description="Empty kibana cache s3 bucket",
Uuid="f7d4f730-4ee1-13e8-9c2d-fa7ae06bbebc",
HandlerPath=os.path.join(dirname, "../helpers/s3_bucket_empty.py"),
BucketName=kibana_bucket.bucket_name,
ResourcePolicies=kibana_bucket_empty_policy,
)
# tag the lambda
core.Tags.of(kibana_bucket_empty).add("project", constants["PROJECT_TAG"])
# needs a dependency
kibana_bucket_empty.node.add_dependency(kibana_bucket)
# kibana lambda update policies
kibana_lambda_update_policy = [
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"s3:ListBucket",
"s3:ListAllMyBuckets",
"lambda:ListFunctions",
"lambda:UpdateFunctionConfiguration",
"cloudfront:ListDistributions",
"s3:GetBucketTagging",
"es:ListDomainNames",
"es:DescribeElasticsearchDomain",
],
resources=["*"],
)
]
# create the kibana lambda update
kibana_lambda_update = CustomResource(
self,
"kibana_lambda_update",
Description="Update ENV vars for kibana api lambda",
PhysicalId="kibanaLambdaUpdate",
Uuid="f7d4f230-4ee1-07e8-9c2d-fa7ae06bbebc",
HandlerPath=os.path.join(dirname, "../helpers/lambda_env_update.py"),
ResourcePolicies=kibana_lambda_update_policy,
)
# tag the lambda
core.Tags.of(kibana_lambda_update).add("project", constants["PROJECT_TAG"])
# needs a dependency
kibana_lambda_update.node.add_dependency(kibana_bucket)
kibana_lambda_update.node.add_dependency(kibana_distribution)
core.CfnOutput(
self,
"kibana_link",
value=f"https://{kibana_distribution.domain_name}/_plugin/kibana",
description="Kibana Web Url",
export_name="kibana-link",
)
def __init__(self, scope: cdk.Construct, construct_id: str,
stack_log_level: str, sales_event_bkt, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Add your stack resources below):
########################################
####### #######
####### Stream Data Producer #######
####### #######
########################################
# Read Lambda Code
try:
with open(
"stacks/back_end/serverless_s3_producer_stack/lambda_src/stream_data_producer.py",
encoding="utf-8",
mode="r",
) as f:
data_producer_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
data_producer_fn = _lambda.Function(
self,
"streamDataProducerFn",
function_name=f"data_producer_{construct_id}",
description="Produce streaming data events and push to S3 stream",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(data_producer_fn_code),
handler="index.lambda_handler",
timeout=cdk.Duration.seconds(2),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"APP_ENV": "Production",
"STORE_EVENTS_BKT": f"{sales_event_bkt.bucket_name}",
"TRIGGER_RANDOM_DELAY": "True"
},
)
# Grant our Lambda Producer privileges to write to S3
sales_event_bkt.grant_read_write(data_producer_fn)
data_producer_fn_version = data_producer_fn.latest_version
data_producer_fn_version_alias = _lambda.Alias(
self,
"streamDataProducerFnAlias",
alias_name="MystiqueAutomation",
version=data_producer_fn_version,
)
# Create Custom Loggroup for Producer
data_producer_lg = _logs.LogGroup(
self,
"streamDataProducerFnLogGroup",
log_group_name=f"/aws/lambda/{data_producer_fn.function_name}",
removal_policy=cdk.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY,
)
# Restrict Produce Lambda to be invoked only from the stack owner account
data_producer_fn.add_permission(
"restrictLambdaInvocationToOwnAccount",
principal=_iam.AccountRootPrincipal(),
action="lambda:InvokeFunction",
source_account=cdk.Aws.ACCOUNT_ID,
# source_arn=sales_event_bkt.bucket_arn
)
self.data_producer_fn_role = data_producer_fn.role
###########################################
################# OUTPUTS #################
###########################################
output_0 = cdk.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page.",
)
output_1 = cdk.CfnOutput(
self,
"LambdaConsumer",
# "StoreOrdersEventsProducer",
value=
f"https://console.aws.amazon.com/lambda/home?region={cdk.Aws.REGION}#/functions/{data_producer_fn.function_name}",
description="Produce streaming data events and push to S3 Topic.",
)
output_2 = cdk.CfnOutput(
self,
"LambdaConsumerRoleArn",
value=f"{self.data_producer_fn_role.role_arn}",
description="StoreOrdersEventsProducerRole",
)
def __init__(
self,
scope: core.Construct,
id: str,
domain_arn: str,
aws_cli_profile: str = None,
cw_trigger_sns_arn_list: list = [],
enable_es_api_output: bool = False,
es_api_output_sns_arn: str = None,
**kwargs,
) -> None:
super().__init__(scope, id, **kwargs)
# Configuring certain aspects of the stack based on the ES domain details
self.configure(domain_arn, aws_cli_profile, cw_trigger_sns_arn_list)
# Setting a Cloudwatch Alarm on the ClusterStatus.red metric
self.create_cw_alarm_with_action(
"ClusterStatus.red",
1,
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1,
1,
"max",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the ClusterStatus.yellow metric
self.create_cw_alarm_with_action(
"ClusterStatus.yellow",
1,
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1,
1,
"max",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the FreeStorageSpace metric. The threshold is 25% of the current volume size (in MB) of a data node.
self.create_cw_alarm_with_action(
"FreeStorageSpace",
self._volume_size * 0.25 * 1000,
cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
1,
1,
"min",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the ClusterIndexWritesBlocked metric
self.create_cw_alarm_with_action(
"ClusterIndexWritesBlocked",
1,
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
5,
1,
"max",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the Nodes metric
self.create_cw_alarm_with_action(
"Nodes",
self._node_count,
cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
1440,
1,
"min",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the AutomatedSnapshotFailure metric
self.create_cw_alarm_with_action(
"AutomatedSnapshotFailure",
1,
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1,
1,
"max",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the CPUUtilization metric
self.create_cw_alarm_with_action(
"CPUUtilization",
80,
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
15,
3,
"avg",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the JVMMemoryPressure metric
self.create_cw_alarm_with_action(
"JVMMemoryPressure",
80,
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
5,
3,
"max",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the MasterCPUUtilization & MasterJVMMemoryPressure metrics
# only if Dedicated Master is enabled
if self._is_dedicated_master_enabled:
self.create_cw_alarm_with_action(
"MasterCPUUtilization",
50,
cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
15,
3,
"avg",
self._sns_topic_list,
)
self.create_cw_alarm_with_action(
"MasterJVMMemoryPressure",
80,
cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
15,
1,
"max",
self._sns_topic_list,
)
# Setting a Cloudwatch Alarm on the KMSKeyError & KMSKeyInaccessible metrics
# only if Encryption at Rest config is enabled
if self._is_encryption_at_rest_enabled:
self.create_cw_alarm_with_action(
"KMSKeyError",
1,
cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1,
1,
"max",
self._sns_topic_list,
)
self.create_cw_alarm_with_action(
"KMSKeyInaccessible",
1,
cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1,
1,
"max",
self._sns_topic_list,
)
if enable_es_api_output:
# Creating a Lambda function to invoke ES _cat APIs corresponding to the triggered CW Alarm
if self._is_vpc_domain:
self._lambda_vpc = ec2.Vpc.from_lookup(self,
self._vpc,
vpc_id=self._vpc)
self._lambda_security_group = ec2.SecurityGroup.from_security_group_id(
self,
self._security_group,
security_group_id=self._security_group)
self._lambda_security_group.connections.allow_internally(
port_range=ec2.Port.tcp(443),
description=
"Ingress rule that allows the aws_es_cw_alarms Lambda to talk to a VPC based ES domain"
)
self._lambda_func = _lambda.Function(
self,
"CWAlarmHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="lambda_function.lambda_handler",
timeout=core.Duration.minutes(15),
environment={
"DOMAIN_ENDPOINT": self._domain_endpoint,
"DOMAIN_ARN": domain_arn
},
vpc=self._lambda_vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE),
security_group=self._lambda_security_group)
else:
self._lambda_func = _lambda.Function(
self,
"CWAlarmHandler",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("lambda"),
handler="lambda_function.lambda_handler",
timeout=core.Duration.minutes(15),
environment={
"DOMAIN_ENDPOINT": self._domain_endpoint,
"DOMAIN_ARN": domain_arn
},
)
# A Custom IAM Policy statement to grant _cat API access to the Lambda function
self._es_policy_statement = iam.PolicyStatement(
actions=["es:ESHttpHead", "es:ESHttpGet"],
effect=iam.Effect.ALLOW,
resources=[domain_arn + "/*"],
)
self._lambda_func.add_to_role_policy(self._es_policy_statement)
# Attaching a SNS topic provided by the user as the trigger for the Lambda function
# If more than one SNS topic is provided, we will attach just the first SNS topic as the trigger
self._lambda_func.add_event_source(
_lambda_event_source.SnsEventSource(self._sns_topic_list[0]))
if es_api_output_sns_arn:
self._lambda_func.add_environment("SNS_TOPIC_ARN",
es_api_output_sns_arn)
# Adding SNS Publish permission since the Lambda function is configured to post
# the output of _cat APIs to the same SNS topic that triggers the function
self._sns_publish_policy_statement = iam.PolicyStatement(
actions=["SNS:Publish"],
effect=iam.Effect.ALLOW,
resources=[es_api_output_sns_arn],
)
self._lambda_func.add_to_role_policy(
self._sns_publish_policy_statement)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
table = ddb.Table(self,
'StripeSampleCustomers',
partition_key={
'name': 'CustomerID',
'type': ddb.AttributeType.STRING
})
bus = events.EventBus(self,
'stripeAppEventBus',
event_bus_name='stripeAppEventBus')
lambda_role_for_go = iam.Role(
self,
"Role",
role_name='stripeAppRole',
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"service-role/AWSLambdaBasicExecutionRole"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonEventBridgeFullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"SecretsManagerReadWrite")
])
customer_created_handler = _lambda.Function(
self,
"createStripeCustomerHandler",
runtime=_lambda.Runtime.GO_1_X,
code=_lambda.Code.asset('lambda/stripe-create-customer'),
handler='createCustomerHandler',
timeout=core.Duration.seconds(8),
role=lambda_role_for_go,
environment={
'CUSTOMER_TABLE_NAME': table.table_name,
})
table.grant_read_write_data(customer_created_handler)
go_lambda = _lambda.Function(
self,
"stripeWebhookEventHandler",
runtime=_lambda.Runtime.GO_1_X,
code=_lambda.Code.asset('lambda/stripe-webhook-handler'),
handler='stripeWebhookHandler',
timeout=core.Duration.seconds(8),
role=lambda_role_for_go,
)
_apigw.LambdaRestApi(self, "stripeWebhookAPI", handler=go_lambda)
customer_created_handler.add_permission(
"createStripeCustomerHandlerPermission",
principal=iam.ServicePrincipal("events.amazonaws.com"),
action='lambda:InvokeFunction',
source_arn=go_lambda.function_arn)
go_lambda.add_permission(
"stripeWebhookHandlerPermission",
principal=iam.ServicePrincipal("lambda.amazonaws.com"),
action='lambda:InvokeFunction',
source_arn=customer_created_handler.function_arn)
event = events.Rule(
self,
'stripeWebhookEventRule',
rule_name='stripeWebhookEventRule',
enabled=True,
event_bus=bus,
description=
'all success events are caught here and logged centrally',
event_pattern=events.EventPattern(
detail={"stripeEvent": ["customer.subscription.created"]},
source=["stripeWebHookHandler.lambda"]))
event.add_target(targets.LambdaFunction(customer_created_handler))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
aws_region = os.environ.get("CDK_DEPLOY_REGION",
os.environ["CDK_DEFAULT_REGION"])
account_id = os.environ.get("CDK_DEPLOY_ACCOUNT",
os.environ["CDK_DEFAULT_ACCOUNT"])
ssm_client = boto3.client('ssm', aws_region)
# Prepare pipeline config details in SSM parameters
if prefix == 'us':
self.qs_reports_env_config = {
"Permissions": [{
"Group_Name": "Critical",
"Reports": ["Sales Results - Critical"]
}, {
"Group_Name":
"HighlyConfidential",
"Reports": [
"Field Operations Dashboard",
"Sales Results - Highly Confidential"
]
}, {
"Group_Name": "BI-developer",
"Reports": ["all"]
}, {
"Group_Name": "BI-Admin",
"Reports": ["all"]
}, {
"Group_Name": "Power-reader",
"Reports": ["read-all"]
}]
}
if prefix == 'eu':
self.qs_reports_env_config = {
"Permissions": [{
"Group_Name": "EU-Critical",
"Reports": ["EUResults - Critical"]
}, {
"Group_Name": "BI-developer",
"Reports": ["all"]
}, {
"Group_Name": "BI-Admin",
"Reports": ["all"]
}, {
"Group_Name":
"EU-HighlyConfidential",
"Reports": [
"EUField Operations Dashboard",
"EUResults - Highly Confidential"
]
}, {
"Group_Name": "Power-reader",
"Reports": ["read-all"]
}]
}
self.qs_reports_env_config_ssm = ssm.StringParameter(
self,
'/qs/config/access',
string_value=json.dumps(self.qs_reports_env_config),
parameter_name='/qs/config/access')
#group-user mapping information is stored in s3 bucket. A ssm parameter stores the bucket name.
self.qs_user_group_config = {
'bucket-name': f'qs-granular-access-demo-{account_id}'
}
self.qs_user_group_config_ssm = ssm.StringParameter(
self,
'/qs/config/groups',
string_value=json.dumps(self.qs_user_group_config),
parameter_name='/qs/config/groups')
# group-role mapping information is stored in a ssm parameter.
self.qs_role_config = {
'BI-developer': 'AUTHOR',
'BI-Admin': 'ADMIN',
'Power-reader': 'AUTHOR',
'Critical': 'READER',
'HighlyConfidential': 'READER'
}
self.qs_role_config_ssm = ssm.StringParameter(
self,
'/qs/config/roles',
string_value=json.dumps(self.qs_role_config),
parameter_name='/qs/config/roles')
# group-namespace mapping information is stored in a ssm parameter.
self.qs_ns_config = {
'BI-developer': 'default',
'BI-Admin': 'default',
'Power-reader': 'default',
'Critical': 'default',
'HighlyConfidential': 'default'
}
self.qs_ns_config_ssm = ssm.StringParameter(
self,
'/qs/config/ns',
string_value=json.dumps(self.qs_ns_config),
parameter_name='/qs/config/ns')
lambda_role = iam.Role(
self,
id='lambda-role',
description='Role for the quicksight lambda',
role_name=f'{aws_region}-role-quicksight-lambda',
max_session_duration=core.Duration.seconds(3600),
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
inline_policies={
'AllowS3Access':
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"kms:GetParametersForImport", "kms:GetPublicKey",
"kms:ListKeyPolicies", "kms:ListRetirableGrants",
"kms:GetKeyPolicy", "kms:ListResourceTags",
"kms:ListGrants", "kms:GetParametersForImport",
"kms:GetKeyRotationStatus", "kms:DescribeKey",
"kms:CreateGrant", "kms:ListAliases",
"kms:ListKeys", "kms:DescribeCustomKeyStores",
"ssm:GetParameters", "ssm:GetParameter",
"ssm:GetParametersByPath"
],
resources=['*']),
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=[
"lambda:InvokeFunction", "logs:CreateLogStream",
"logs:CreateLogGroup", "logs:PutLogEvents",
"quicksight:*", "s3:HeadBucket",
"s3:ListAllMyBuckets", "s3:PutObject",
"s3:GetObject", "s3:ListBucket",
"s3:GetObjectVersionForReplication",
"s3:GetBucketPolicy", "s3:GetObjectVersion",
"cloudwatch:PutMetricData", "sts:GetCallerIdentity"
],
resources=['*'])
])
})
user_init = _lambda.Function(
self,
'user_init',
handler='user_init.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset(
os.path.join(current_dir, '../lambda_functions/user_init/')),
function_name='user_init',
role=lambda_role,
timeout=core.Duration.minutes(15),
memory_size=512)
check_team_members = _lambda.Function(
self,
'check_team_members',
handler='check_team_members.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset(
os.path.join(current_dir,
'../lambda_functions/check_team_members/')),
function_name='check_team_members',
role=lambda_role,
timeout=core.Duration.minutes(15),
memory_size=512,
environment={'aws_region': f'{core.Aws.REGION}'})
downgrade_user = _lambda.Function(
self,
'downgrade_user',
handler='downgrade_user.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_8,
code=_lambda.Code.from_asset(
os.path.join(current_dir,
'../lambda_functions/downgrade_user/')),
function_name='downgrade_user',
role=lambda_role,
timeout=core.Duration.minutes(15),
memory_size=2048,
environment={'aws_region': f'{core.Aws.REGION}'})
granular_access = _lambda.Function(
self,
'granular_access',
handler='granular_access.lambda_handler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.from_asset(
os.path.join(current_dir,
'../lambda_functions/granular_access/')),
function_name='granular_access',
role=lambda_role,
timeout=core.Duration.minutes(15),
memory_size=2048,
environment={'aws_region': f'{core.Aws.REGION}'})
quicksight_event_rule = events.Rule(
self,
'QuickSightCWEventRule',
description=
'CloudWatch rule to detect new QuickSight user creation',
rule_name='qs-gc-user-creation',
targets=[targets.LambdaFunction(user_init)],
event_pattern=events.EventPattern(
source=['aws.quicksight'],
detail_type=['AWS Service Event via CloudTrail'],
detail={
"eventSource": ["quicksight.amazonaws.com"],
"eventName": ["CreateUser"]
}))
quicksight_schedule_rule = events.Rule(
self,
'quicksight_schedule_rule',
description=
'CloudWatch rule to run QS objects/groups assignment every hour',
rule_name='qs-gc-every-hour',
schedule=events.Schedule.cron(minute="0"),
targets=[
targets.LambdaFunction(check_team_members),
targets.LambdaFunction(granular_access)
])
quicksight_assume_condition_object = {
"StringEquals": {
"SAML:aud": "https://signin.aws.amazon.com/saml"
}
}
quicksight_federated_prin_with_conditionb_obj = iam.FederatedPrincipal(
f'arn:aws:iam::{core.Aws.ACCOUNT_ID}:saml-provider/saml',
quicksight_assume_condition_object, 'sts:AssumeRoleWithSAML')
quicksight_resource_scope = '${aws:userid}'
quicksight_reader_saml_inline_policies = {
'AllowQuicksightAccessSAML':
iam.PolicyDocument(statements=[
iam.PolicyStatement(
effect=iam.Effect.ALLOW,
actions=['quicksight:CreateReader'],
resources=[
f'arn:aws:quicksight::{core.Aws.ACCOUNT_ID}:user/{quicksight_resource_scope}'
])
])
}
quicksight_users = iam.Role(
self,
id=
f"quicksight-fed-{prefix}-users", # this is the default group with no access
description='Role for the quicksight reader SAML',
role_name=f"quicksight-fed-{prefix}-users",
max_session_duration=core.Duration.seconds(3600),
assumed_by=quicksight_federated_prin_with_conditionb_obj,
inline_policies=quicksight_reader_saml_inline_policies)
def __init__(self, scope: core.Construct, id: str, props,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/Vpc.html
vpc = ec2.Vpc(self,
"vpc",
cidr=props['vpc_CIDR'],
max_azs=3,
subnet_configuration=[{
'cidrMask': 28,
'name': 'public',
'subnetType': ec2.SubnetType.PUBLIC
}, {
'cidrMask':
28,
'name':
'private',
'subnetType':
ec2.SubnetType.PRIVATE
}, {
'cidrMask':
28,
'name':
'db',
'subnetType':
ec2.SubnetType.ISOLATED
}])
rds_subnetGroup = rds.SubnetGroup(
self,
"rds_subnetGroup",
description=
f"Group for {props['environment']}-{props['application']}-{props['unit']} DB",
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_rds/DatabaseCluster.html
##TODO:ADD Aurora Serverless Option
rds_instance = rds.DatabaseCluster(
self,
'wordpress-db',
engine=rds.DatabaseClusterEngine.aurora_mysql(
version=rds.AuroraMysqlEngineVersion.VER_2_07_2),
instances=1,
instance_props=rds.InstanceProps(
vpc=vpc,
enable_performance_insights=props[
'rds_enable_performance_insights'],
instance_type=ec2.InstanceType(
instance_type_identifier=props['rds_instance_type'])),
subnet_group=rds_subnetGroup,
storage_encrypted=props['rds_storage_encrypted'],
backup=rds.BackupProps(retention=core.Duration.days(
props['rds_automated_backup_retention_days'])))
EcsToRdsSeurityGroup = ec2.SecurityGroup(
self,
"EcsToRdsSeurityGroup",
vpc=vpc,
description="Allow WordPress containers to talk to RDS")
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html
db_cred_generator = _lambda.Function(
self,
'db_creds_generator',
runtime=_lambda.Runtime.PYTHON_3_8,
handler='db_creds_generator.handler',
code=_lambda.Code.asset('lambda/db_creds_generator'),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.ISOLATED
), #vpc.select_subnets(subnet_type = ec2.SubnetType("ISOLATED")).subnets ,
environment={
'SECRET_NAME': rds_instance.secret.secret_name,
})
#Set Permissions and Sec Groups
rds_instance.connections.allow_from(
EcsToRdsSeurityGroup,
ec2.Port.tcp(3306)) #Open hole to RDS in RDS SG
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html
file_system = efs.FileSystem(
self,
"MyEfsFileSystem",
vpc=vpc,
encrypted=True, # file system is not encrypted by default
lifecycle_policy=props['efs_lifecycle_policy'],
performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
throughput_mode=efs.ThroughputMode.BURSTING,
removal_policy=core.RemovalPolicy(props['efs_removal_policy']),
enable_automatic_backups=props['efs_automatic_backups'])
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/Cluster.html?highlight=ecs%20cluster#aws_cdk.aws_ecs.Cluster
cluster = ecs.Cluster(
self,
"Cluster",
vpc=vpc,
container_insights=props['ecs_enable_container_insights'])
if props['deploy_bastion_host']:
#ToDo: Deploy bastion host with a key file
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ec2/BastionHostLinux.html
bastion_host = ec2.BastionHostLinux(self, 'bastion_host', vpc=vpc)
rds_instance.connections.allow_from(bastion_host,
ec2.Port.tcp(3306))
#######################
### Developer Tools ###
# SFTP into the EFS Shared File System
NetToolsSecret = secretsmanager.Secret(
self,
"NetToolsSecret",
generate_secret_string=secretsmanager.SecretStringGenerator(
secret_string_template=json.dumps({
"username": '******',
"ip": ''
}),
generate_string_key="password",
exclude_characters='/"'))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_efs/FileSystem.html#aws_cdk.aws_efs.FileSystem.add_access_point
AccessPoint = file_system.add_access_point(
"access-point",
path="/",
create_acl=efs.Acl(
owner_uid=
"100", #https://aws.amazon.com/blogs/containers/developers-guide-to-using-amazon-efs-with-amazon-ecs-and-aws-fargate-part-2/
owner_gid="101",
permissions="0755"))
EfsVolume = ecs.Volume(
name="efs",
efs_volume_configuration=ecs.EfsVolumeConfiguration(
file_system_id=file_system.file_system_id,
transit_encryption="ENABLED",
authorization_config=ecs.AuthorizationConfig(
access_point_id=AccessPoint.access_point_id)))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateTaskDefinition.html
NetToolsTask = ecs.FargateTaskDefinition(self,
"TaskDefinition",
cpu=256,
memory_limit_mib=512,
volumes=[EfsVolume])
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateTaskDefinition.html#aws_cdk.aws_ecs.FargateTaskDefinition.add_container
NetToolsContainer = NetToolsTask.add_container(
"NetTools",
image=ecs.ContainerImage.from_registry('netresearch/sftp'),
command=['test:test:100:101:efs'])
NetToolsContainer.add_port_mappings(
ecs.PortMapping(container_port=22, protocol=ecs.Protocol.TCP))
NetToolsContainer.add_mount_points(
ecs.MountPoint(
container_path=
"/home/test/efs", #ToDo build path out with username from secret
read_only=False,
source_volume=EfsVolume.name,
))
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_ecs/FargateService.html?highlight=fargateservice#aws_cdk.aws_ecs.FargateService
service = ecs.FargateService(
self,
"Service",
cluster=cluster,
task_definition=NetToolsTask,
platform_version=ecs.FargatePlatformVersion(
"VERSION1_4"), #Required for EFS
)
#ToDo somehow store container's IP on deploy
#Allow traffic to EFS Volume from Net Tools container
service.connections.allow_to(file_system, ec2.Port.tcp(2049))
#ToDo allow bastion host into container on port 22
#https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html
bastion_ip_locator = _lambda.Function(
self,
'bastion_ip_locator',
function_name=
f"{props['environment']}-{props['application']}-{props['unit']}-SFTP-IP",
runtime=_lambda.Runtime.PYTHON_3_8,
handler='bastion_ip_locator.handler',
code=_lambda.Code.asset('lambda/bastion_ip_locator'),
environment={
'CLUSTER_NAME': cluster.cluster_arn,
'SERVICE_NAME': service.service_name
})
#Give needed perms to bastion_ip_locator for reading info from ECS
bastion_ip_locator.add_to_role_policy(
iam.PolicyStatement(
actions=["ecs:DescribeTasks"],
resources=[
#f"arn:aws:ecs:us-east-1:348757191778:service/{cluster.cluster_name}/{service.service_name}",
f"arn:aws:ecs:us-east-1:348757191778:task/{cluster.cluster_name}/*"
]))
bastion_ip_locator.add_to_role_policy(
iam.PolicyStatement(actions=[
"ecs:ListTasks",
],
resources=["*"],
conditions={
'ArnEquals': {
'ecs:cluster': cluster.cluster_arn
}
}))
self.output_props = props.copy()
self.output_props["vpc"] = vpc
self.output_props["rds_instance"] = rds_instance
self.output_props["EcsToRdsSeurityGroup"] = EcsToRdsSeurityGroup
self.output_props["file_system"] = file_system
self.output_props["cluster"] = cluster
def __init__(self, scope: core.Construct, id: str,
api_gateway: aws_apigateway.RestApi, **kwargs):
super().__init__(scope, id, **kwargs)
raw_bucket = aws_s3.Bucket(self,
'DataPipelineRawBucket',
public_read_access=False,
removal_policy=core.RemovalPolicy.DESTROY)
processed_bucket = aws_s3.Bucket(
self,
'DataPipelineProcessedBucket',
public_read_access=False,
removal_policy=core.RemovalPolicy.DESTROY)
# Simple 1 step data pipeline - raw_bucket => lambda_processor => processed_bucket
ecr_image = aws_lambda.EcrImageCode.from_asset_image(
directory=os.path.join(os.getcwd(),
"startuptoolbag/app/lambda-s3-processor"))
lambda_processor = aws_lambda.Function(
self,
id="lambdaS3DataProcessor",
description="Processes Data Landed In S3",
code=ecr_image,
handler=aws_lambda.Handler.FROM_IMAGE,
runtime=aws_lambda.Runtime.FROM_IMAGE,
environment={
'RAW_BUCKET': raw_bucket.bucket_name,
'PROCESSED_BUCKET': processed_bucket.bucket_name
},
memory_size=128,
reserved_concurrent_executions=1,
timeout=core.Duration.seconds(900))
notification = aws_s3_notifications.LambdaDestination(lambda_processor)
raw_bucket.add_event_notification(aws_s3.EventType.OBJECT_CREATED,
notification)
raw_bucket.grant_read(lambda_processor)
processed_bucket.grant_read_write(lambda_processor)
# API Lambda which is backed by data in S3
# This is essentially an Object Lambda - https://aws.amazon.com/blogs/aws/introducing-amazon-s3-object-lambda-use-your-code-to-process-data-as-it-is-being-retrieved-from-s3/
# TODO investigate when CDK support ObjectLambda or CDK Solutions Patterns
ecr_image = aws_lambda.EcrImageCode.from_asset_image(
directory=os.path.join(os.getcwd(),
"startuptoolbag/app/lambda-s3-server"))
lambda_handler = aws_lambda.Function(
self,
id="lambdaS3Server",
description="Handle API requests backed by S3",
code=ecr_image,
handler=aws_lambda.Handler.FROM_IMAGE,
runtime=aws_lambda.Runtime.FROM_IMAGE,
environment={'BUCKET': processed_bucket.bucket_name},
memory_size=128,
reserved_concurrent_executions=1,
timeout=core.Duration.seconds(10))
processed_bucket.grant_read(lambda_handler)
foo_r = api_gateway.root.add_resource("data")
foo_r.add_method('GET',
aws_apigateway.LambdaIntegration(lambda_handler))
def __init__(self, scope: core.Construct, construct_id: str,
stack_log_level: str, back_end_api_name: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Add your stack resources below):
###################################################
####### #######
####### Customer Info Data Producer #######
####### #######
###################################################
# Read Lambda Code
try:
with open(
"stacks/back_end/serverless_api_producer_cust_info/lambda_src/api_producer_cust_info.py",
encoding="utf-8",
mode="r") as f:
data_producer_fn_code = f.read()
except OSError:
print("Unable to read Lambda Function Code")
raise
data_producer_fn = _lambda.Function(
self,
"sqsDataProducerFn",
function_name=f"data_producer_fn_{construct_id}",
description=
"Produce customer info messages with attributes and send them as a list",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(data_producer_fn_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(2),
reserved_concurrent_executions=2,
environment={
"LOG_LEVEL": f"{stack_log_level}",
"APP_ENV": "Production",
"MAX_MSGS_TO_PRODUCE": "2",
})
data_producer_fn_version = data_producer_fn.latest_version
data_producer_fn_version_alias = _lambda.Alias(
self,
"greeterFnAlias",
alias_name="MystiqueAutomation",
version=data_producer_fn_version)
# Create Custom Loggroup for Producer
data_producer_lg = _logs.LogGroup(
self,
"dataProducerLogGroup",
log_group_name=f"/aws/lambda/{data_producer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
# Restrict Produce Lambda to be invoked only from the stack owner account
data_producer_fn.add_permission(
"restrictLambdaInvocationToFhInOwnAccount",
principal=_iam.AccountRootPrincipal(),
action="lambda:InvokeFunction",
source_account=core.Aws.ACCOUNT_ID)
# Create API Gateway
# Add API GW front end for the Lambda
cust_info_api_stage_01_options = _apigw.StageOptions(
stage_name="miztiik-cust-info-v1",
logging_level=_apigw.MethodLoggingLevel.INFO)
cust_info_api = _apigw.RestApi(
self,
"backEnd01Api",
rest_api_name=f"{back_end_api_name}",
deploy_options=cust_info_api_stage_01_options,
endpoint_types=[_apigw.EndpointType.EDGE],
description=
f"{GlobalArgs.OWNER}: API Security Automation demonstration using - Mutual TLS"
)
cust_info_api_res_root = cust_info_api.root.add_resource("cust-info")
cust_info_res_v1 = cust_info_api_res_root.add_resource("v1")
cust_info_v1_data = cust_info_res_v1.add_resource("data")
cust_info_v1_data_method_get = cust_info_v1_data.add_method(
http_method="GET",
request_parameters={
"method.request.header.InvocationType": True,
"method.request.path.number": True
},
integration=_apigw.LambdaIntegration(handler=data_producer_fn,
proxy=True))
###########################################
################# OUTPUTS #################
###########################################
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{GlobalArgs.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
"CustomerInfoDataProducer",
value=
f"https://console.aws.amazon.com/lambda/home?region={core.Aws.REGION}#/functions/{data_producer_fn.function_name}",
description="Produce data events and push to SQS Queue.")
output_2 = core.CfnOutput(
self,
"CustomerInfoProducerApi",
value=f"{cust_info_v1_data.url}",
description="Use your browser to fetch customer data from this API."
)
def __init__(self, scope: core.Construct, id: str, wiki_api_endpoint,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
# DynamodDB Table(TODO:Create re-usable data model):
queries_table = _dynamodb.Table(
self,
"queriesDataTable",
partition_key=_dynamodb.Attribute(
name="_id", type=_dynamodb.AttributeType.STRING))
# Create AWS XRay Layer
aws_xray_layer = _lambda.LayerVersion(
self,
'awsXrayLayer',
code=_lambda.Code.from_asset(
'lambda_src/layer_code/aws_xray_python_37.zip'),
compatible_runtimes=[
_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8
],
license=
f'Mystique LambdaLayer of AWS XRay, Refer to AWS for license.',
description='Layer to trace AWS Lamba Calls')
# Create Requests Layer
requests_layer = _lambda.LayerVersion(
self,
'requestsLayer',
code=_lambda.Code.from_asset(
'lambda_src/layer_code/requests_python_37.zip'),
compatible_runtimes=[
_lambda.Runtime.PYTHON_3_7, _lambda.Runtime.PYTHON_3_8
],
description='Python requests Layer to make HTTP calls')
# Defines an AWS Lambda resource
with open("lambda_src/polyglot_strangler_fig_svc.py",
encoding="utf8") as fp:
polyglot_svc_fn_handler_code = fp.read()
polyglot_svc_fn = _lambda.Function(
self,
id='polyglotStranglerFigService',
function_name="polyglot_svc_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(polyglot_svc_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(59),
environment={
'LD_LIBRARY_PATH': '/opt/python',
'WIKI_API_ENDPOINT': wiki_api_endpoint,
'DDB_TABLE_NAME': queries_table.table_name,
'TRIGGER_RANDOM_FAILURES': 'True'
},
layers=[aws_xray_layer, requests_layer],
tracing=_lambda.Tracing.ACTIVE)
# Grant Lambda permissions to write to Dynamodb
queries_table.grant_read_write_data(polyglot_svc_fn)
##### PUBLISH TO API GW ######
# Enable AWS XRay Tracing at API GW
polyglot_svc_api_stage_options = _apigw.StageOptions(
stage_name="myst",
logging_level=_apigw.MethodLoggingLevel.INFO,
data_trace_enabled=True,
metrics_enabled=True,
tracing_enabled=True)
# Create API Gateway
api_01 = _apigw.RestApi(self,
'polglotApiEndpoint',
rest_api_name='mystique-xray-tracer-api',
deploy_options=polyglot_svc_api_stage_options)
v1 = api_01.root.add_resource("polyglot_svc")
# Add resource for HTTP Endpoint: API Hosted on EC2
polyglot_svc_api_resource_00 = v1.add_resource('wiki')
self.polyglot_svc_api_resource_01 = polyglot_svc_api_resource_00.add_resource(
'{query}')
polyglot_svc_api_lambda_integration = _apigw.LambdaIntegration(
handler=polyglot_svc_fn,
proxy=True,
integration_responses=[{
"statusCode": "200"
}],
request_parameters={
"integration.request.path.query": "method.request.path.query"
})
self.polyglot_svc_api_resource_01.add_method(
http_method="GET",
integration=polyglot_svc_api_lambda_integration,
method_responses=[{
"statusCode": "200"
}],
request_parameters={
'method.request.header.Content-Type': False,
'method.request.path.query': True
})
##### MONITORING ######
# Now let us create alarms for our Lambda Function
# alarm is raised there are more than 5(threshold) of the measured metrics in two(datapoint) of the last three seconds(evaluation):
# Period=60Seconds, Eval=3, Threshold=5
# metric_errors(): How many invocations of this Lambda fail.
# https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_lambda/Function.html
polyglot_svc_fn_error_alarm = polyglot_svc_fn.metric_errors(
).create_alarm(
self,
"polglotSvcAlarm",
alarm_name="polyglot_svc_fn_error_alarm",
threshold=10,
evaluation_periods=2,
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_THRESHOLD,
period=core.Duration.minutes(1),
treat_missing_data=_cloudwatch.TreatMissingData.NOT_BREACHING)
# SNS For Alerts for Polyglot Service
polyglot_svc_support_topic = _sns.Topic(
self,
"polyglotSvcTopic",
display_name="PolyglotSvc",
topic_name="polyglotSvcSupportTopic")
# Subscribe Polyglot Service Team Email to topic
for email in global_args.POLYGLOT_SUPPORT_EMAIL:
polyglot_svc_support_topic.add_subscription(
_subs.EmailSubscription(email_address=email))
# polyglot_svc_support_topic.add_subscription(
# _subs.EmailSubscription(global_args.POLYGLOT_SUPPORT_EMAIL))
# Add the topic to the Alarm
polyglot_svc_fn_error_alarm.add_alarm_action(
_cw_actions.SnsAction(polyglot_svc_support_topic))
# Create CloudWatch Dashboard for Polyglot Service Team
polyglot_svc_dashboard = _cloudwatch.Dashboard(
self, id="polyglotSvcDashboard", dashboard_name="Polyglot-Svc")
polyglot_svc_fn_invocation_metric = polyglot_svc_fn.metric_invocations(
label="Invocations",
period=core.Duration.minutes(1),
statistic="Sum")
polyglot_svc_dashboard.add_widgets(
_cloudwatch.AlarmWidget(title="Lambda-Errors",
alarm=polyglot_svc_fn_error_alarm))
polyglot_svc_dashboard.add_widgets(
# Lambda Metrics
# TODO: here monitor all lambda concurrency not just the working one. Limitation from CDK
# Lambda now supports monitor single lambda concurrency, will change this after CDK support
_cloudwatch.GraphWidget(
title="Lambda-all-concurrent",
left=[
polyglot_svc_fn.metric_all_concurrent_executions(
statistic="Sum",
period=core.Duration.minutes(1),
color=_cloudwatch.Color.GREEN)
]),
_cloudwatch.GraphWidget(
title="Lambda-invocations/errors/throttles",
left=[
polyglot_svc_fn.metric_invocations(
statistic="Sum", period=core.Duration.minutes(1)),
polyglot_svc_fn.metric_errors(
statistic="Sum",
period=core.Duration.minutes(1),
color=_cloudwatch.Color.RED),
polyglot_svc_fn.metric_throttles(
statistic="Sum",
period=core.Duration.minutes(1),
color=_cloudwatch.Color.ORANGE)
]),
_cloudwatch.GraphWidget(title="Lambda-duration",
left=[
polyglot_svc_fn.metric_duration(
statistic="Average",
period=core.Duration.minutes(1))
]),
# _cloudwatch.Row(_cloudwatch.TextWidget(markdown="# XRay Profiler KPI")),
# _cloudwatch.Row(_cloudwatch.Spacer()),
# DynamoDB Metrics
_cloudwatch.Row(
_cloudwatch.GraphWidget(
title="DynamoDB-Write-Capacity-Units",
left=[
queries_table.metric_consumed_write_capacity_units(
statistic="Sum", period=core.Duration.minutes(1))
]),
_cloudwatch.GraphWidget(
title="DynamoDB-Read-Capacity-Units",
left=[
queries_table.metric_consumed_read_capacity_units(
statistic="Sum", period=core.Duration.minutes(1))
])),
)
###########################################
################# OUTPUTS #################
###########################################
output_0 = core.CfnOutput(
self,
"AutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output_1 = core.CfnOutput(
self,
'PolyglotServiceApiUrl',
value=f'{self.polyglot_svc_api_resource_01.url}',
description=
f'Call the polyglot API, replace <query> with your search term')
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
s3_org_data = _s3.Bucket(self,
ORIGINAL_DATA_BUCKET_NAME,
bucket_name=ORIGINAL_DATA_BUCKET_NAME,
removal_policy=core.RemovalPolicy.RETAIN)
s3_transformed_data = _s3.Bucket(
self,
TRANSFORMED_DATA_BUCKET_NAME,
bucket_name=TRANSFORMED_DATA_BUCKET_NAME,
removal_policy=core.RemovalPolicy.RETAIN)
# title-read
s3_deployment.BucketDeployment(
self,
"s3-deployment-{}".format(TITLE_READ),
sources=[
s3_deployment.Source.asset("data/{}/".format(TITLE_READ))
],
destination_bucket=s3_org_data,
destination_key_prefix="{}/".format(TITLE_READ))
# title
s3_deployment.BucketDeployment(
self,
"s3-deployment-{}".format(TITLE),
sources=[s3_deployment.Source.asset("data/{}/".format(TITLE))],
destination_bucket=s3_org_data,
destination_key_prefix="{}/".format(TITLE))
# user
s3_deployment.BucketDeployment(
self,
"s3-deployment-{}".format(USER),
sources=[s3_deployment.Source.asset("data/{}/".format(USER))],
destination_bucket=s3_org_data,
destination_key_prefix="{}/".format(USER))
statement = iam.PolicyStatement(actions=[
"s3:*", "glue:*", "iam:ListRolePolicies", "iam:GetRole",
"iam:GetRolePolicy"
],
resources=["*"])
write_to_s3_policy = iam.PolicyDocument(statements=[statement])
glue_role = iam.Role(
self,
'GlueCrawlerRole-dna',
role_name='GlueCrawlerRole-dna',
inline_policies=[write_to_s3_policy],
assumed_by=iam.ServicePrincipal('glue.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSGlueServiceRole')
])
#TODO add IAM role for ctas lambda
dna_database = glue.Database(self,
"dna-glue-database-id",
database_name=GLUE_DATABASE_NAME)
# create glue table
title_read_table = glue.Table(
self,
"{}-table-id".format(TITLE_READ),
table_name="{}_table".format(TITLE_READ).replace("-", "_"),
database=dna_database,
columns=[{
"name": "USER_ID",
"type": glue.Schema.STRING
}, {
"name": "ITEM_ID",
"type": glue.Schema.STRING
}, {
"name": "TIMESTAMP",
"type": glue.Schema.BIG_INT
}, {
"name": "TITLE",
"type": glue.Schema.STRING
}, {
"name": "EVENT_TYPE",
"type": glue.Schema.STRING
}],
data_format=glue.DataFormat.CSV,
bucket=s3_org_data,
s3_prefix=TITLE_READ)
title_table = glue.Table(self,
"{}-table-id".format(TITLE),
table_name="{}_table".format(TITLE).replace(
"-", "_"),
database=dna_database,
columns=[{
"name": "ITEM_ID",
"type": glue.Schema.STRING
}, {
"name": "CREATION_TIMESTAMP",
"type": glue.Schema.BIG_INT
}, {
"name": "TITLE",
"type": glue.Schema.STRING
}, {
"name": "TAG",
"type": glue.Schema.STRING
}],
data_format=glue.DataFormat.CSV,
bucket=s3_org_data,
s3_prefix=TITLE)
user_table = glue.Table(self,
"{}-table-id".format(USER),
table_name="{}_table".format(USER).replace(
"-", "_"),
database=dna_database,
columns=[
{
"name": "USER_ID",
"type": glue.Schema.STRING
},
{
"name": "NAME",
"type": glue.Schema.STRING
},
{
"name": "EMAIL",
"type": glue.Schema.STRING
},
{
"name": "GENDER",
"type": glue.Schema.STRING,
"categorical": True
},
{
"name": "AGE",
"type": glue.Schema.BIG_INT,
"categorical": True
},
],
data_format=glue.DataFormat.CSV,
bucket=s3_org_data,
s3_prefix=USER)
_athena.CfnWorkGroup(self,
"athena_workgroup_id",
name=ATHENA_WORKGROUP)
ctas_lambda_trigger = _event.Rule(
self,
"ctas-lambda-trigger-event-id",
rule_name="ctas-lambda-trigger-event",
schedule=_event.Schedule.cron(minute="10", hour="*"))
s3_statement = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
# resources = [s3_bucket.bucket_arn, "{}/*".format(s3_bucket.bucket_arn)],
resources=["*"],
actions=["s3:*"])
athena_statement = iam.PolicyStatement(
effect=iam.Effect.ALLOW,
resources=["*"],
actions=["athena:StartQueryExecution", "glue:*"])
ctas_lambda_func = _lambda.Function(
self,
"CTAS_query",
function_name="CTAS_query",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset("src/lambda"),
handler="ctas_lambda.lambda_handler",
description="CTAS query to transform AVRO file, batch execution",
environment={
"BUCKET_NAME": s3_transformed_data.bucket_name,
"DATABASE_NAME": GLUE_DATABASE_NAME,
"ATHENA_WORKGROUP": ATHENA_WORKGROUP
},
timeout=core.Duration.minutes(3))
ctas_lambda_func.add_to_role_policy(s3_statement)
ctas_lambda_func.add_to_role_policy(athena_statement)
ctas_lambda_trigger.add_target(
_target.LambdaFunction(ctas_lambda_func))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Add your stack resources below):
# Create Kinesis Data Stream
stream_data_pipe = _kinesis.Stream(
self,
"streamDataPipe",
retention_period=core.Duration.hours(24),
shard_count=1,
stream_name="data_pipe")
# Create an S3 Bucket for storing streaming data events
stream_data_store = _s3.Bucket(
self, "streamDataLake", removal_policy=core.RemovalPolicy.DESTROY)
# Read Lambda Code
try:
with open(
"advanced_use_cases/lambda_src/stream_record_consumer.py",
mode="r") as f:
stream_consumer_fn_code = f.read()
except OSError:
print("Unable to read lambda function code")
# Deploy the lambda function
stream_consumer_fn = _lambda.Function(
self,
"streamConsumerFn",
function_name="stream_consumer_fn",
description=
"Process streaming data events from kinesis and store in S3",
runtime=_lambda.Runtime.PYTHON_3_7,
handler="index.lambda_handler",
code=_lambda.InlineCode(stream_consumer_fn_code),
timeout=core.Duration.seconds(3),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"BUCKET_NAME": f"{stream_data_store.bucket_name}"
})
# Update Lambda Permissions To Use Stream
stream_data_pipe.grant_read(stream_consumer_fn)
# Add permissions to lambda to write to S3
roleStmt1 = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[f"{stream_data_store.bucket_arn}/*"],
actions=["s3:PutObject"])
roleStmt1.sid = "AllowLambdaToWriteToS3"
stream_consumer_fn.add_to_role_policy(roleStmt1)
# Create Custom Loggroup for Consumer
stream_consumer_lg = _logs.LogGroup(
self,
"streamConsumerLogGroup",
log_group_name=f"/aws/lambda/{stream_consumer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
# Create New Kinesis Event Source
stream_data_pipe_event_source = _lambda_event_sources.KinesisEventSource(
stream=stream_data_pipe,
starting_position=_lambda.StartingPosition.LATEST,
batch_size=1)
# Attach Kinesis Event Source To Lambda
stream_consumer_fn.add_event_source(stream_data_pipe_event_source)
########################################
####### #######
####### Stream Data Producer #######
####### #######
########################################
# Read Lambda Code
try:
with open("advanced_use_cases/lambda_src/stream_data_producer.py",
mode="r") as f:
data_producer_fn_code = f.read()
except OSError:
print("Unable to read lambda function code")
# Deploy the lambda function
data_producer_fn = _lambda.Function(
self,
"streamDataProducerFn",
function_name="data_producer_fn",
description=
"Produce streaming data events and push to Kinesis stream",
runtime=_lambda.Runtime.PYTHON_3_7,
handler="index.lambda_handler",
code=_lambda.InlineCode(data_producer_fn_code),
timeout=core.Duration.seconds(60),
reserved_concurrent_executions=1,
environment={
"LOG_LEVEL": "INFO",
"STREAM_NAME": f"{stream_data_pipe.stream_name}"
})
# Grant our Lambda Producer privileges to write to Kinesis Data Stream
stream_data_pipe.grant_read_write(data_producer_fn)
# Create Custom Loggroup for Producer
data_producer_lg = _logs.LogGroup(
self,
"dataProducerLogGroup",
log_group_name=f"/aws/lambda/{data_producer_fn.function_name}",
removal_policy=core.RemovalPolicy.DESTROY,
retention=_logs.RetentionDays.ONE_DAY)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# 创建sts lambda访问角色
# action -> statement -> policy -> role -> attach lambda
actions = [
"logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents",
"sts:AssumeRole"
]
policyStatement = PolicyStatement(actions=actions, effect=Effect.ALLOW)
policyStatement.add_all_resources()
policy_name = "{}-policy".format(Constant.PROJECT_NAME)
sts_policy = Policy(self, policy_name, policy_name=policy_name)
sts_policy.add_statements(policyStatement)
role_name = "{}-role".format(Constant.PROJECT_NAME)
access_role = Role(self,
role_name,
role_name=role_name,
assumed_by=ServicePrincipal('lambda.amazonaws.com'))
sts_policy.attach_to_role(access_role)
# 创建thum lambda访问角色
# action -> statement -> policy -> role
thum_actions = [
"logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents",
"s3:PutObject"
]
thum_policyStatement = PolicyStatement(actions=thum_actions,
effect=Effect.ALLOW)
thum_policyStatement.add_all_resources()
thum_policy_name = "{}-policy-thum".format(Constant.PROJECT_NAME)
thum_policy = Policy(self,
thum_policy_name,
policy_name=thum_policy_name)
thum_policy.add_statements(thum_policyStatement)
thum_role_name = "{}-role-thum".format(Constant.PROJECT_NAME)
thum_access_role = Role(
self,
thum_role_name,
role_name=thum_role_name,
assumed_by=ServicePrincipal('lambda.amazonaws.com'))
thum_policy.attach_to_role(thum_access_role)
# 创建S3 put的角色
# action -> statement -> policy -> role
s3_actions = [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket",
"s3:PutObjectTagging",
"s3:PutObjectAcl",
]
s3_policyStatement = PolicyStatement(actions=s3_actions,
effect=Effect.ALLOW)
s3_policyStatement.add_all_resources()
s3_policy_name = "{}-policy-s3".format(Constant.PROJECT_NAME)
s3_policy = Policy(self, s3_policy_name, policy_name=s3_policy_name)
s3_policy.add_statements(s3_policyStatement)
s3_role_name = "{}-role-s3".format(Constant.PROJECT_NAME)
s3_access_role = Role(self,
s3_role_name,
role_name=s3_role_name,
assumed_by=ArnPrincipal(access_role.role_arn))
s3_policy.attach_to_role(s3_access_role)
# 创建STS lambda
sts_lambda = _lambda.Function(
self,
'sts',
function_name='sts',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='auth.handler',
timeout=Duration.minutes(1),
role=access_role,
)
sts_lambda.add_environment("role_to_assume_arn",
s3_access_role.role_arn)
base_api = apigw.RestApi(
self,
'Endpoint',
endpoint_types=[EndpointType.REGIONAL],
)
example_entity = base_api.root.add_resource('auth')
example_entity_lambda_integration = apigw.LambdaIntegration(
sts_lambda,
proxy=False,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin':
"'*'",
}
}])
example_entity.add_method(
'GET',
example_entity_lambda_integration,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}])
self.add_cors_options(example_entity)
# 创建缩略图lambda
layer_cv2 = _lambda.LayerVersion(
self,
'cv2',
code=_lambda.Code.from_bucket(
s3.Bucket.from_bucket_name(self,
"cdk-data-layer",
bucket_name='nowfox'),
'cdk-data/cv2.zip'),
compatible_runtimes=[_lambda.Runtime.PYTHON_3_7],
)
lambda_thum = _lambda.Function(
self,
'thum',
function_name='thum',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='thum.handler',
timeout=Duration.minutes(1),
role=thum_access_role,
)
lambda_thum.add_environment("frame_second", "3")
lambda_thum.add_layers(layer_cv2)
# 创建存储上传图片的bucket
s3_bucket_name = "{}-{}".format("upload", self._get_UUID(4))
s3_upload = s3.Bucket(
self,
id=s3_bucket_name,
bucket_name=s3_bucket_name,
# access_control=s3.BucketAccessControl.PUBLIC_READ,#不建议使用这个，这个会有list权限，下面这个没有list权限
public_read_access=True,
removal_policy=core.RemovalPolicy.
DESTROY, # TODO: destroy for test
# removal_policy=core.RemovalPolicy.RETAIN
)
notification = aws_s3_notifications.LambdaDestination(lambda_thum)
s3_filter = s3.NotificationKeyFilter(suffix=".mp4")
s3_upload.add_event_notification(s3.EventType.OBJECT_CREATED_PUT,
notification, s3_filter)
s3_upload.add_cors_rule(allowed_methods=[
HttpMethods.POST, HttpMethods.PUT, HttpMethods.GET
],
allowed_origins=["*"],
allowed_headers=["*"])
'''
#创建上传lambda
lambda_upload = _lambda.Function(
self, 'upload',function_name='upload',
runtime=_lambda.Runtime.JAVA_8,
code=_lambda.Code.from_bucket(s3.Bucket.from_bucket_name(self, "cdk-data-upload-jar", bucket_name='nowfox'),
'cdk-data/fileupload.jar'),
handler='fileupload.FileUploadFunctionHandler::handleRequest',
timeout=Duration.minutes(1),
memory_size=512,
role=access_role,
)
lambda_upload.add_environment("bucket", s3_bucket_name)
lambda_upload.add_environment("region", "cn-northwest-1")
'''
core.CfnOutput(self,
"authUrl",
value=base_api.url + "auth",
description="authUrl")
core.CfnOutput(self,
"S3BucketName",
value=s3_bucket_name,
description="S3BucketName")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
lambda_greetings = _lambda.Function(self,
'GreetingsHandler',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='greetings.greet',
memory_size=128)
api_stage_options = api_gateway.StageOptions(stage_name="dev")
api = api_gateway.LambdaRestApi(self,
'Endpoint',
handler=lambda_greetings,
deploy_options=api_stage_options)
# create dynamo table
demo_table = aws_dynamodb.Table(
self,
"ToDos_table",
partition_key=aws_dynamodb.Attribute(
name="id", type=aws_dynamodb.AttributeType.STRING))
get_todos_handler = _lambda.Function(
self,
'GetTodos',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='todos.get',
memory_size=128)
get_todos_handler.add_environment("TABLE_NAME", demo_table.table_name)
demo_table.grant_read_data(get_todos_handler)
get_todos_integration = api_gateway.LambdaIntegration(
get_todos_handler)
#/todos GET = todo listeleyecek
todos = api.root.add_resource("todos")
todos.add_method("GET", get_todos_integration)
get_todo_handler = _lambda.Function(self,
'GetTodo',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='todo.get',
memory_size=128)
get_todo_handler.add_environment("TABLE_NAME", demo_table.table_name)
demo_table.grant_read_data(get_todo_handler)
put_todo_handler = _lambda.Function(self,
'PutTodo',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='todo.put',
memory_size=128)
put_todo_handler.add_environment("TABLE_NAME", demo_table.table_name)
demo_table.grant_write_data(put_todo_handler)
delete_todo_handler = _lambda.Function(
self,
'DeleteTodo',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='todo.delete',
memory_size=128)
delete_todo_handler.add_environment("TABLE_NAME",
demo_table.table_name)
demo_table.grant_write_data(delete_todo_handler)
get_todo_integration = api_gateway.LambdaIntegration(get_todo_handler)
put_todo_integration = api_gateway.LambdaIntegration(put_todo_handler)
delete_todo_integration = api_gateway.LambdaIntegration(
delete_todo_handler)
#/todo path
todo = api.root.add_resource("todo")
#/todo/{todoId}
todo_gd = todo.add_resource("{todoId}")
#/todo/{todoId} GET
todo_gd.add_method("GET", get_todo_integration)
#/todo/{todoId} DELETE = todo silecek
todo_gd.add_method("DELETE", delete_todo_integration)
#/todo/new
todo_p = todo.add_resource("new")
todo_p.add_method("PUT", put_todo_integration)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
bucket_with_sns = s3.Bucket(self,
"bucket-with-sns-integration",
removal_policy=core.RemovalPolicy.DESTROY)
bucket_with_lambda = s3.Bucket(
self,
"bucket-with-lambda-integration",
removal_policy=core.RemovalPolicy.DESTROY)
exchange_topic = sns.Topic(self, "lambda-info-topic")
bucket_with_sns.add_event_notification(
event=s3.EventType.OBJECT_CREATED,
dest=s3_notifications.SnsDestination(exchange_topic))
measurement_table = dynamodb.Table(
self,
"measurement-table",
partition_key=dynamodb.Attribute(
name="PK", type=dynamodb.AttributeType.STRING),
sort_key=dynamodb.Attribute(name="SK",
type=dynamodb.AttributeType.STRING),
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
removal_policy=core.RemovalPolicy.DESTROY)
s3_event_generator = _lambda.Function(
self,
"s3-event-generator",
code=_lambda.Code.from_asset(
path=os.path.join(os.path.dirname(__file__), "..", "src")),
environment={
"BUCKET_WITH_LAMBDA": bucket_with_lambda.bucket_name,
"BUCKET_WITH_SNS": bucket_with_sns.bucket_name,
},
handler="s3_event_generator.lambda_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
timeout=core.Duration.seconds(300),
memory_size=1024,
)
bucket_with_lambda.grant_write(s3_event_generator)
bucket_with_sns.grant_write(s3_event_generator)
measure_lambda = _lambda.Function(
self,
"measure-lambda",
code=_lambda.Code.from_asset(
path=os.path.join(os.path.dirname(__file__), "..", "src")),
environment={"MEASUREMENT_TABLE": measurement_table.table_name},
handler="measure_lambda.lambda_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
timeout=core.Duration.seconds(50),
memory_size=1024,
)
measurement_table.grant_read_write_data(measure_lambda)
measure_lambda.add_event_source(
lambda_event_sources.SnsEventSource(exchange_topic))
measure_lambda.add_event_source(
lambda_event_sources.S3EventSource(
bucket=bucket_with_lambda,
events=[s3.EventType.OBJECT_CREATED]))
ssm.StringParameter(self,
"bucket-with-sns-parameter",
string_value=bucket_with_sns.bucket_name,
parameter_name=BUCKET_WITH_SNS_PARAMETER)
ssm.StringParameter(self,
"bucket-with-lambda-parameter",
string_value=bucket_with_lambda.bucket_name,
parameter_name=BUCKET_WITH_LAMBDA_PARAMETER)
ssm.StringParameter(self,
"measurement-table-parameter",
string_value=measurement_table.table_name,
parameter_name=MEASUREMENT_TABLE_PARAMETER)
ssm.StringParameter(self,
"generator-function-name-parameter",
string_value=s3_event_generator.function_name,
parameter_name=GENERATOR_FUNCTION_NAME_PARAMETER)
def __init__(
self,
scope: cdk.Construct,
id: str,
cognito_user_pool: cognito.UserPool,
s3_bucket_name: str,
create_configuration_lambda_role_arn: str,
redis: ec.CfnCacheCluster,
domain_name: str,
**kwargs
) -> None:
super().__init__(scope, id, **kwargs)
config_yaml = yaml.load(open("config.yaml"), Loader=yaml.FullLoader)
spoke_accounts = config_yaml.get("spoke_accounts", [])
cognito_user_pool_client = cognito.UserPoolClient(
self,
"UserPoolClient",
user_pool=cognito_user_pool,
generate_secret=True,
supported_identity_providers=[
cognito.UserPoolClientIdentityProvider.COGNITO
],
prevent_user_existence_errors=True,
o_auth=cognito.OAuthSettings(
callback_urls=[
"https://" + domain_name + "/auth",
"https://" + domain_name + "/oauth2/idpresponse",
],
logout_urls=["https://" + domain_name + "/logout"],
flows=cognito.OAuthFlows(
authorization_code_grant=True, implicit_code_grant=True
),
scopes=[cognito.OAuthScope.OPENID, cognito.OAuthScope.EMAIL],
),
auth_flows=cognito.AuthFlow(user_password=True, user_srp=True),
)
describe_cognito_user_pool_client = cr.AwsCustomResource(
self,
"UserPoolClientIDResource",
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(
resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE
),
on_create=cr.AwsSdkCall(
service="CognitoIdentityServiceProvider",
action="describeUserPoolClient",
parameters={
"UserPoolId": cognito_user_pool.user_pool_id,
"ClientId": cognito_user_pool_client.user_pool_client_id,
},
physical_resource_id=cr.PhysicalResourceId.of(
cognito_user_pool_client.user_pool_client_id
),
),
install_latest_aws_sdk=True,
log_retention=logs.RetentionDays.ONE_WEEK,
)
cognito_user_pool_client_secret = (
describe_cognito_user_pool_client.get_response_field(
"UserPoolClient.ClientSecret"
)
)
imported_create_configuration_lambda_role = iam.Role.from_role_arn(
self,
"ImportedCreateConfigurationFileLambdaRole",
role_arn=create_configuration_lambda_role_arn,
)
jwt_secret = config_yaml["jwt_secret"]
config_secret_dict = {
"oidc_secrets": {
"client_id": cognito_user_pool_client.user_pool_client_id,
"secret": cognito_user_pool_client_secret,
"client_scope": ["email", "openid"],
},
"jwt_secret": jwt_secret,
}
config_secret_yaml = yaml.dump(
config_secret_dict,
explicit_start=True,
default_flow_style=False,
)
config_secret = cr.AwsCustomResource(
self,
"ConfigSecretResource",
policy=cr.AwsCustomResourcePolicy.from_sdk_calls(
resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE
),
on_update=cr.AwsSdkCall(
service="SecretsManager",
action="updateSecret",
parameters={
"SecretId": CONFIG_SECRET_NAME,
"SecretString": config_secret_yaml,
},
physical_resource_id=cr.PhysicalResourceId.from_response("Name"),
),
on_create=cr.AwsSdkCall(
service="SecretsManager",
action="createSecret",
parameters={
"Name": CONFIG_SECRET_NAME,
"Description": "Sensitive configuration parameters for ConsoleMe",
"SecretString": config_secret_yaml,
},
physical_resource_id=cr.PhysicalResourceId.from_response("Name"),
),
on_delete=cr.AwsSdkCall(
service="SecretsManager",
action="deleteSecret",
parameters={
"SecretId": CONFIG_SECRET_NAME,
"ForceDeleteWithoutRecovery": True,
},
),
install_latest_aws_sdk=True,
log_retention=logs.RetentionDays.ONE_WEEK,
)
create_configuration_lambda = lambda_.Function(
self,
"CreateConfigurationFileLambda",
code=lambda_.Code.from_asset("resources/create_config_lambda"),
handler="index.handler",
timeout=cdk.Duration.seconds(30),
layers=[create_dependencies_layer(self, "create_config_lambda")],
runtime=lambda_.Runtime.PYTHON_3_8,
role=imported_create_configuration_lambda_role,
environment={
"DEPLOYMENT_BUCKET": s3_bucket_name,
"OIDC_METADATA_URL": "https://cognito-idp."
+ self.region
+ ".amazonaws.com/"
+ cognito_user_pool.user_pool_id
+ "/.well-known/openid-configuration",
"REDIS_HOST": redis.attr_redis_endpoint_address,
"SES_IDENTITY_ARN": "arn:aws:ses:"
+ self.region
+ ":"
+ self.account
+ ":identity/"
+ domain_name,
"SUPPORT_CHAT_URL": "https://discord.gg/nQVpNGGkYu",
"APPLICATION_ADMIN": "consoleme_admin",
"ACCOUNT_NUMBER": self.account,
"ISSUER": domain_name,
"SPOKE_ACCOUNTS": ",".join(spoke_accounts),
"CONFIG_SECRET_NAME": CONFIG_SECRET_NAME,
},
)
create_configuration_resource_provider = cr.Provider(
self,
"CreateConfigurationFileProvider",
on_event_handler=create_configuration_lambda,
log_retention=logs.RetentionDays.ONE_WEEK,
)
create_configuration_lambda_resource = cdk.CustomResource(
self,
"CreateConfigurationFile",
service_token=create_configuration_resource_provider.service_token,
removal_policy=cdk.RemovalPolicy.DESTROY,
properties={"UUID": str(uuid4())},
)
create_configuration_lambda_resource.node.add_dependency(config_secret)
def __init__(self, scope: core.Construct, construct_id: str, configs, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# The code that defines your stack goes here
bucket = s3.Bucket(self, "ScraperBucket-{}".format(configs['stage']),
bucket_name = 'scraper-{}-{}'.format(configs['stage'],configs['aws_account'])
)
scraper_fn = aws_lambda.Function(self, "ScraperFn-{}".format(configs['stage']),
function_name = "scraper-{}".format(configs['stage']),
code=aws_lambda.Code.from_asset("src/fanduelscraper",
bundling={
"image": aws_lambda.Runtime.PYTHON_3_8.bundling_docker_image,
"command": ["bash", "-c",
"pip install -r requirements.txt -t /asset-output && cp -au index.py src /asset-output"
]
}
),
handler="index.scraper",
runtime=aws_lambda.Runtime.PYTHON_3_8,
timeout = core.Duration.seconds(300),
memory_size = 1048,
environment={
"BUCKET_NAME": bucket.bucket_name
}
)
scraper_handler_fn = aws_lambda.Function(self, "ScraperHandlerFn-{}".format(configs['stage']),
function_name = "scraper-handler-{}".format(configs['stage']),
code=aws_lambda.Code.from_asset("src/fanduelscraper",
bundling={
"image": aws_lambda.Runtime.PYTHON_3_8.bundling_docker_image,
"command": ["bash", "-c",
"pip install -r requirements.txt -t /asset-output && cp -au index.py src /asset-output"
]
}
),
handler="index.scraper_handler",
runtime=aws_lambda.Runtime.PYTHON_3_8,
timeout = core.Duration.seconds(300),
memory_size = 1048,
environment={
"SCRAPER_FN": scraper_fn.function_name
}
)
#Make sure to give lambda permission to write to S3 bucket
bucket.grant_write(scraper_fn)
scraper_fn.grant_invoke(scraper_handler_fn)
if(configs['stage'] == 'prod'):
rule = events.Rule(self, "ScraperCronRule-{}".format(configs['stage']),
schedule=events.Schedule.cron(
minute='0',
hour='*',
month='*',
week_day='*',
year='*'),
)
rule.add_target(targets.LambdaFunction(scraper_handler_fn))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
demo_vpc = aws_ec2.Vpc(self,
"network",
cidr="10.10.0.0/16",
max_azs=2,
subnet_configuration=[])
demo_subnets = []
demo_subnets.append(
aws_ec2.Subnet(self,
'sbn-demo-1',
availability_zone=demo_vpc.availability_zones[0],
vpc_id=demo_vpc.vpc_id,
cidr_block='10.10.0.0/25'))
demo_subnets.append(
aws_ec2.Subnet(self,
'sbn-demo-2',
availability_zone=demo_vpc.availability_zones[1],
vpc_id=demo_vpc.vpc_id,
cidr_block='10.10.0.128/25'))
demo_vpc.add_interface_endpoint(
'secretmanager',
service=aws_ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER,
subnets=aws_ec2.SubnetSelection(subnets=demo_subnets))
db_subnet_group = aws_rds.SubnetGroup(
self,
'sbng-demo-rds',
description='demo db subnet group',
vpc=demo_vpc,
removal_policy=core.RemovalPolicy.DESTROY,
vpc_subnets=aws_ec2.SubnetSelection(subnets=demo_subnets))
db_security_group = aws_ec2.SecurityGroup(self,
'sg-demo-rds',
vpc=demo_vpc)
db_security_group.add_ingress_rule(
peer=aws_ec2.Peer.ipv4('10.10.0.0/16'),
connection=aws_ec2.Port(
protocol=aws_ec2.Protocol.TCP,
string_representation="to allow from the vpc internal",
from_port=3306,
to_port=3306))
mysql_instance = aws_rds.DatabaseInstance(
self,
'mys-demo-rds',
engine=aws_rds.DatabaseInstanceEngine.MYSQL,
vpc=demo_vpc,
vpc_subnets=aws_ec2.SubnetSelection(subnets=demo_subnets),
security_groups=[db_security_group],
iam_authentication=True)
db_secret = mysql_instance.secret
role_init_db = aws_iam.Role(
self,
'cmd_role_init_src_db',
assumed_by=aws_iam.ServicePrincipal("lambda.amazonaws.com"))
lambda_base_policy_statement = aws_iam.PolicyStatement(
resources=['*'],
actions=[
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents',
"ec2:CreateNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DeleteNetworkInterface",
])
role_init_db.add_to_policy(lambda_base_policy_statement)
role_init_db.add_to_policy(
aws_iam.PolicyStatement(resources=[db_secret.secret_arn],
actions=[
"secretsmanager:GetResourcePolicy",
"secretsmanager:GetSecretValue",
"secretsmanager:DescribeSecret",
"secretsmanager:ListSecretVersionIds"
]))
db_user = '******'
func_init_db = aws_lambda.Function(
self,
'func_init_db',
function_name='demo-rds_func_init_db',
handler='handler.init',
runtime=aws_lambda.Runtime.PYTHON_3_8,
code=aws_lambda.Code.asset('./app_stack/func_init_db'),
role=role_init_db,
timeout=core.Duration.seconds(10),
allow_public_subnet=False,
vpc=demo_vpc,
vpc_subnets=aws_ec2.SubnetSelection(subnets=demo_subnets),
environment={
'db_secret': db_secret.secret_name,
'db_user': db_user
})
# When the role_name specified as static value, it could be occurring with KMS related exception.
# It's because re-created lambda function with static name of role would be refer to the role which is not exsisted.
# https://github.com/serverless/examples/issues/279
role_test_db = aws_iam.Role(
self,
'demo_role_test_db',
assumed_by=aws_iam.ServicePrincipal("lambda.amazonaws.com"),
role_name=db_user)
role_test_db.add_to_policy(lambda_base_policy_statement)
role_test_db.add_to_policy(
aws_iam.PolicyStatement(resources=[
'arn:aws:rds-db:%s:%s:dbuser:*/*' % (REGION, ACCOUNT)
],
actions=[
"rds-db:connect",
]))
role_test_db.add_to_policy(
aws_iam.PolicyStatement(resources=[db_secret.secret_arn],
actions=[
"secretsmanager:GetResourcePolicy",
"secretsmanager:GetSecretValue",
"secretsmanager:DescribeSecret",
"secretsmanager:ListSecretVersionIds"
]))
func_test_db = aws_lambda.Function(
self,
'func_test_db',
function_name='demo-rds_test_iam_auth',
handler='handler.init',
runtime=aws_lambda.Runtime.PYTHON_3_8,
code=aws_lambda.Code.asset('./app_stack/func_test_db'),
role=role_test_db,
timeout=core.Duration.seconds(10),
allow_public_subnet=False,
vpc=demo_vpc,
vpc_subnets=aws_ec2.SubnetSelection(subnets=demo_subnets),
environment={
'db_secret': db_secret.secret_name,
'db_user': db_user
})
def __init__(
self,
scope: core.Construct,
id: str,
dataset_metadata_filename: str,
memory: int = 1024,
timeout: int = 30,
concurrent: int = 100,
code_dir: str = "./",
**kwargs: Any,
) -> None:
"""Define stack."""
super().__init__(scope, id, **kwargs)
# add cache
if config.VPC_ID:
vpc = ec2.Vpc.from_lookup(
self,
f"{id}-vpc",
vpc_id=config.VPC_ID,
)
else:
vpc = ec2.Vpc(self, f"{id}-vpc")
sb_group = escache.CfnSubnetGroup(
self,
f"{id}-subnet-group",
description=f"{id} subnet group",
subnet_ids=[sb.subnet_id for sb in vpc.private_subnets],
)
lambda_function_security_group = ec2.SecurityGroup(self,
f"{id}-lambda-sg",
vpc=vpc)
lambda_function_security_group.add_egress_rule(
ec2.Peer.any_ipv4(),
connection=ec2.Port(protocol=ec2.Protocol("ALL"),
string_representation=""),
description="Allow lambda security group all outbound access",
)
cache_security_group = ec2.SecurityGroup(self,
f"{id}-cache-sg",
vpc=vpc)
cache_security_group.add_ingress_rule(
lambda_function_security_group,
connection=ec2.Port(protocol=ec2.Protocol("ALL"),
string_representation=""),
description=
"Allow Lambda security group access to Cache security group",
)
cache = escache.CfnCacheCluster(
self,
f"{id}-cache",
cache_node_type=config.CACHE_NODE_TYPE,
engine=config.CACHE_ENGINE,
num_cache_nodes=config.CACHE_NODE_NUM,
vpc_security_group_ids=[cache_security_group.security_group_id],
cache_subnet_group_name=sb_group.ref,
)
logs_access = iam.PolicyStatement(
actions=[
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
],
resources=["*"],
)
ec2_network_access = iam.PolicyStatement(
actions=[
"ec2:CreateNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DeleteNetworkInterface",
],
resources=["*"],
)
lambda_env = DEFAULT_ENV.copy()
lambda_env.update(
dict(
MODULE_NAME="dashboard_api.main",
VARIABLE_NAME="app",
WORKERS_PER_CORE="1",
LOG_LEVEL="error",
MEMCACHE_HOST=cache.attr_configuration_endpoint_address,
MEMCACHE_PORT=cache.attr_configuration_endpoint_port,
DATASET_METADATA_FILENAME=dataset_metadata_filename,
))
lambda_function_props = dict(
runtime=aws_lambda.Runtime.PYTHON_3_7,
code=self.create_package(code_dir),
handler="handler.handler",
memory_size=memory,
timeout=core.Duration.seconds(timeout),
environment=lambda_env,
security_groups=[lambda_function_security_group],
vpc=vpc,
)
if concurrent:
lambda_function_props[
"reserved_concurrent_executions"] = concurrent
lambda_function = aws_lambda.Function(self, f"{id}-lambda",
**lambda_function_props)
lambda_function.add_to_role_policy(s3_full_access_to_data_bucket)
lambda_function.add_to_role_policy(logs_access)
lambda_function.add_to_role_policy(ec2_network_access)
# defines an API Gateway Http API resource backed by our "dynamoLambda" function.
api = apigw.HttpApi(
self,
f"{id}-endpoint",
default_integration=apigw_integrations.LambdaProxyIntegration(
handler=lambda_function),
)
core.CfnOutput(self, "API Endpoint", value=api.url)
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# create s3 bucket
s3_Bucket = s3.Bucket(self,
"AWS-Cookbook-Recipe-403",
removal_policy=RemovalPolicy.DESTROY)
aws_s3_deployment.BucketDeployment(
self,
'S3Deployment',
destination_bucket=s3_Bucket,
sources=[aws_s3_deployment.Source.asset("./s3_content")],
retain_on_delete=False)
isolated_subnets = ec2.SubnetConfiguration(
name="ISOLATED",
subnet_type=ec2.SubnetType.PRIVATE_ISOLATED,
cidr_mask=24)
# create VPC
vpc = ec2.Vpc(self,
'AWS-Cookbook-VPC',
cidr='10.10.0.0/23',
subnet_configuration=[isolated_subnets])
vpc.add_interface_endpoint(
'VPCSecretsManagerInterfaceEndpoint',
service=ec2.InterfaceVpcEndpointAwsService(
'secretsmanager'
), # Find names with - aws ec2 describe-vpc-endpoint-services | jq '.ServiceNames'
private_dns_enabled=True,
subnets=ec2.SubnetSelection(
one_per_az=False, subnet_type=ec2.SubnetType.PRIVATE_ISOLATED),
)
vpc.add_gateway_endpoint(
's3GateWayEndPoint',
service=ec2.GatewayVpcEndpointAwsService('s3'),
subnets=[
ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE_ISOLATED)
],
)
subnet_group = rds.SubnetGroup(
self,
'rds_subnet_group',
description='VPC Subnet Group for RDS',
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
one_per_az=False, subnet_type=ec2.SubnetType.PRIVATE_ISOLATED))
rds_security_group = ec2.SecurityGroup(
self,
'rds_security_group',
description='Security Group for the RDS Instance',
allow_all_outbound=True,
vpc=vpc)
db_name = 'AWSCookbookRecipe403'
rds_instance = rds.DatabaseInstance(
self,
'DBInstance',
engine=rds.DatabaseInstanceEngine.mysql(
version=rds.MysqlEngineVersion.VER_5_7_26),
instance_type=ec2.InstanceType("m5.large"),
vpc=vpc,
multi_az=False,
database_name=db_name,
instance_identifier='awscookbook403db',
delete_automated_backups=True,
deletion_protection=False,
removal_policy=RemovalPolicy.DESTROY,
allocated_storage=8,
subnet_group=subnet_group,
security_groups=[rds_security_group])
# mkdir -p lambda-layers/sqlparse/python
# cd layers/sqlparse/python
# pip install sqlparse --target="."
# cd ../../../
# create Lambda Layer
sqlparse = aws_lambda.LayerVersion(
self,
"sqlparse",
code=aws_lambda.AssetCode('lambda-layers/sqlparse'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_8],
description="sqlparse",
license=
"https://github.com/andialbrecht/sqlparse/blob/master/LICENSE")
pymysql = aws_lambda.LayerVersion(
self,
"pymysql",
code=aws_lambda.AssetCode('lambda-layers/pymysql'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_8],
description="pymysql",
license="MIT")
smartopen = aws_lambda.LayerVersion(
self,
"smartopen",
code=aws_lambda.AssetCode('lambda-layers/smart_open'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_8],
description="smartopen",
license="MIT")
lambda_function = aws_lambda.Function(
self,
'LambdaRDS',
code=aws_lambda.AssetCode("./mysql-lambda/"),
handler="lambda_function.lambda_handler",
environment={
"DB_SECRET_ARN": rds_instance.secret.secret_arn,
"S3_BUCKET": s3_Bucket.bucket_name
},
layers=[sqlparse, pymysql, smartopen],
memory_size=1024,
runtime=aws_lambda.Runtime.PYTHON_3_8,
timeout=Duration.seconds(600),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE_ISOLATED))
rds_instance.secret.grant_read(lambda_function)
rds_instance.connections.allow_from(lambda_function.connections,
ec2.Port.tcp(3306), "Ingress")
s3_Bucket.grant_read(lambda_function)
db_app_security_group = ec2.SecurityGroup(
self,
'db_app_security_group',
description='Security Group for the DB App',
allow_all_outbound=True,
vpc=vpc)
db_app_lambda_function = aws_lambda.Function(
self,
'LambdaApp',
code=aws_lambda.AssetCode("./db-app-lambda/"),
handler="lambda_function.lambda_handler",
environment={
"DB_HOST": rds_instance.db_instance_endpoint_address,
},
layers=[pymysql],
memory_size=1024,
security_groups=[db_app_security_group],
runtime=aws_lambda.Runtime.PYTHON_3_8,
timeout=Duration.seconds(600),
vpc=vpc,
vpc_subnets=ec2.SubnetSelection(
subnet_type=ec2.SubnetType.PRIVATE_ISOLATED))
create_params = {
"FunctionName": lambda_function.function_arn,
}
on_create = custom_resources.AwsSdkCall(
action='invoke',
service='Lambda',
parameters=create_params,
physical_resource_id=custom_resources.PhysicalResourceId.of(
'LambdaRDS'))
policy_statement = iam.PolicyStatement(
actions=["lambda:InvokeFunction"],
effect=iam.Effect.ALLOW,
resources=[lambda_function.function_arn],
)
policy = custom_resources.AwsCustomResourcePolicy.from_statements(
statements=[policy_statement])
custom_resources.AwsCustomResource(
self,
'CustomResource',
policy=policy,
on_create=on_create,
log_retention=logs.RetentionDays.TWO_WEEKS)
# outputs
CfnOutput(self, 'VpcId', value=vpc.vpc_id)
CfnOutput(self,
'PyMysqlLambdaLayerArn',
value=pymysql.layer_version_arn)
CfnOutput(self,
'RdsDatabaseId',
value=rds_instance.instance_identifier)
CfnOutput(self,
'RdsSecurityGroup',
value=rds_security_group.security_group_id)
CfnOutput(self, 'DbName', value=db_name)
CfnOutput(self,
'RdsSecretArn',
value=rds_instance.secret.secret_full_arn)
CfnOutput(self,
'RdsEndpoint',
value=rds_instance.db_instance_endpoint_address)
CfnOutput(self,
'RdsPort',
value=rds_instance.db_instance_endpoint_port)
isolated_subnets = vpc.select_subnets(
subnet_type=ec2.SubnetType.PRIVATE_ISOLATED)
CfnOutput(self,
'IsolatedSubnets',
value=', '.join(map(str, isolated_subnets.subnet_ids)))
CfnOutput(self,
'DbAppFunctionRoleName',
value=db_app_lambda_function.role.role_name)
CfnOutput(self,
'DbAppFunctionArn',
value=db_app_lambda_function.function_arn)
CfnOutput(self,
'DbAppFunctionName',
value=db_app_lambda_function.function_name)
CfnOutput(self, 'BucketName', value=s3_Bucket.bucket_name)
CfnOutput(self,
'DbAppFunctionSgId',
value=db_app_security_group.security_group_id)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# -----------------------------------------------------------------------------------------------------------
# The Simple Webservice Logic - This is what we will be monitoring
#
# API GW HTTP API, Lambda Fn and DynamoDB
# https://github.com/cdk-patterns/serverless/tree/master/the-simple-webservice
# -----------------------------------------------------------------------------------------------------------
# DynamoDB Table
table = dynamo_db.Table(
self,
"Hits",
partition_key=dynamo_db.Attribute(
name="path", type=dynamo_db.AttributeType.STRING),
billing_mode=dynamo_db.BillingMode.PAY_PER_REQUEST)
# defines an AWS Lambda resource
dynamo_lambda = _lambda.Function(
self,
"DynamoLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X, # execution environment
handler="lambda.handler", # file is "lambda", function is "handler"
code=_lambda.Code.from_asset(
"lambda"), # Code loaded from the lambda dir
environment={'HITS_TABLE_NAME': table.table_name})
# grant the lambda role read/write permissions to our table'
table.grant_read_write_data(dynamo_lambda)
# defines an API Gateway Http API resource backed by our "dynamoLambda" function.
api = api_gw.HttpApi(self,
'HttpAPI',
default_integration=api_gw.LambdaProxyIntegration(
handler=dynamo_lambda))
core.CfnOutput(self, 'HTTP API Url', value=api.url)
# -----------------------------------------------------------------------------------------------------------
# Monitoring Logic Starts Here
#
# This is everything we need to understand the state of our system:
# - custom metrics
# - cloudwatch alarms
# - custom cloudwatch dashboard
# -----------------------------------------------------------------------------------------------------------
# SNS Topic so we can hook things into our alerts e.g. email
error_topic = sns.Topic(self, 'theBigFanTopic')
###
# Custom Metrics
###
api_gw_4xx_error_percentage = cloud_watch.MathExpression(
expression="m1/m2*100",
label="% API Gateway 4xx Errors",
using_metrics={
"m1":
self.metric_for_api_gw(api.http_api_id, '4XXError',
'4XX Errors', 'sum'),
"m2":
self.metric_for_api_gw(api.http_api_id, 'Count', '# Requests',
'sum'),
},
period=core.Duration.minutes(5))
# Gather the % of lambda invocations that error in past 5 mins
lambda_error_perc = cloud_watch.MathExpression(
expression="e / i * 100",
label="% of invocations that errored, last 5 mins",
using_metrics={
"i":
dynamo_lambda.metric(metric_name="Invocations",
statistic="sum"),
"e":
dynamo_lambda.metric(metric_name="Errors", statistic="sum"),
},
period=core.Duration.minutes(5))
# note: throttled requests are not counted in total num of invocations
lambda_throttled_perc = cloud_watch.MathExpression(
expression="t / (i + t) * 100",
label="% of throttled requests, last 30 mins",
using_metrics={
"i":
dynamo_lambda.metric(metric_name="Invocations",
statistic="sum"),
"t":
dynamo_lambda.metric(metric_name="Throttles", statistic="sum"),
},
period=core.Duration.minutes(5))
# I think usererrors are at an account level rather than a table level so merging
# these two metrics until I can get a definitive answer. I think usererrors
# will always show as 0 when scoped to a table so this is still effectively
# a system errors count
dynamo_db_total_errors = cloud_watch.MathExpression(
expression="m1 + m2",
label="DynamoDB Errors",
using_metrics={
"m1": table.metric_user_errors(),
"m2": table.metric_system_errors(),
},
period=core.Duration.minutes(5))
# Rather than have 2 alerts, let's create one aggregate metric
dynamo_db_throttles = cloud_watch.MathExpression(
expression="m1 + m2",
label="DynamoDB Throttles",
using_metrics={
"m1":
table.metric(metric_name="ReadThrottleEvents",
statistic="sum"),
"m2":
table.metric(metric_name="WriteThrottleEvents",
statistic="sum"),
},
period=core.Duration.minutes(5))
###
# Alarms
###
# Api Gateway
# 4xx are user errors so a large volume indicates a problem
cloud_watch.Alarm(self,
id="API Gateway 4XX Errors > 1%",
metric=api_gw_4xx_error_percentage,
threshold=1,
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
# 5xx are internal server errors so we want 0 of these
cloud_watch.Alarm(self,
id="API Gateway 5XX Errors > 0",
metric=self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="5XXError",
label="5XX Errors",
stat="p99"),
threshold=0,
period=core.Duration.minutes(5),
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
cloud_watch.Alarm(self,
id="API p99 latency alarm >= 1s",
metric=self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="Latency",
label="API GW Latency",
stat="p99"),
threshold=1000,
period=core.Duration.minutes(5),
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
# Lambda
# 2% of Dynamo Lambda invocations erroring
cloud_watch.Alarm(self,
id="Dynamo Lambda 2% Error",
metric=lambda_error_perc,
threshold=2,
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
# 1% of Lambda invocations taking longer than 1 second
cloud_watch.Alarm(self,
id="Dynamo Lambda p99 Long Duration (>1s)",
metric=dynamo_lambda.metric_duration(),
period=core.Duration.minutes(5),
threshold=1000,
evaluation_periods=6,
datapoints_to_alarm=1,
statistic="p99",
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
# 2% of our lambda invocations are throttled
cloud_watch.Alarm(self,
id="Dynamo Lambda 2% Throttled",
metric=lambda_throttled_perc,
threshold=2,
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
# DynamoDB
# DynamoDB Interactions are throttled - indicated poorly provisioned
cloud_watch.Alarm(self,
id="DynamoDB Table Reads/Writes Throttled",
metric=dynamo_db_throttles,
threshold=1,
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
# There should be 0 DynamoDB errors
cloud_watch.Alarm(self,
id="DynamoDB Errors > 0",
metric=dynamo_db_total_errors,
threshold=0,
evaluation_periods=6,
datapoints_to_alarm=1,
treat_missing_data=cloud_watch.TreatMissingData.NOT_BREACHING) \
.add_alarm_action(actions.SnsAction(error_topic))
dashboard = cloud_watch.Dashboard(self, id="CloudWatchDashBoard")
dashboard.add_widgets(
cloud_watch.GraphWidget(title="Requests",
width=8,
left=[
self.metric_for_api_gw(
api_id=api.http_api_id,
metric_name="Count",
label="# Requests",
stat="sum")
]),
cloud_watch.GraphWidget(
title="API GW Latency",
width=8,
stacked=True,
left=[
self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="Latency",
label="API Latency p50",
stat="p50"),
self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="Latency",
label="API Latency p90",
stat="p90"),
self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="Latency",
label="API Latency p99",
stat="p99")
]),
cloud_watch.GraphWidget(
title="API GW Errors",
width=8,
stacked=True,
left=[
self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="4XXError",
label="4XX Errors",
stat="sum"),
self.metric_for_api_gw(api_id=api.http_api_id,
metric_name="5XXError",
label="5XX Errors",
stat="sum")
]),
cloud_watch.GraphWidget(title="Dynamo Lambda Error %",
width=8,
left=[lambda_error_perc]),
cloud_watch.GraphWidget(
title="Dynamo Lambda Duration",
width=8,
stacked=True,
left=[
dynamo_lambda.metric_duration(statistic="p50"),
dynamo_lambda.metric_duration(statistic="p90"),
dynamo_lambda.metric_duration(statistic="p99")
]),
cloud_watch.GraphWidget(title="Dynamo Lambda Throttle %",
width=8,
left=[lambda_throttled_perc]),
cloud_watch.GraphWidget(
title="DynamoDB Latency",
width=8,
stacked=True,
left=[
table.metric_successful_request_latency(
dimensions={
"TableName": table.table_name,
"Operation": "GetItem"
}),
table.metric_successful_request_latency(
dimensions={
"TableName": table.table_name,
"Operation": "UpdateItem"
}),
table.metric_successful_request_latency(
dimensions={
"TableName": table.table_name,
"Operation": "PutItem"
}),
table.metric_successful_request_latency(
dimensions={
"TableName": table.table_name,
"Operation": "DeleteItem"
}),
table.metric_successful_request_latency(
dimensions={
"TableName": table.table_name,
"Operation": "Query"
}),
]),
cloud_watch.GraphWidget(
title="DynamoDB Consumed Read/Write Units",
width=8,
stacked=False,
left=[
table.metric(metric_name="ConsumedReadCapacityUnits"),
table.metric(metric_name="ConsumedWriteCapacityUnits")
]),
cloud_watch.GraphWidget(
title="DynamoDB Throttles",
width=8,
stacked=True,
left=[
table.metric(metric_name="ReadThrottleEvents",
statistic="sum"),
table.metric(metric_name="WriteThrottleEvents",
statistic="sum")
]),
)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# Create repo for Amplify static site
amplify_repo = codecommit.Repository(
self,
'amplify-wild-rydes-repo',
repository_name='amplify-wild-rydes',
description='Repo for the Wild Rydes static site for Amplify')
# Create repo for holding the code for this project
app_repo = codecommit.Repository(
self,
'app-serverless-workshop-repo',
repository_name='app-wild-rydes-serverless-workshop',
description=
'Repo for project from webapp.serverlessworkshops.io/staticwebhosting/overview/'
)
# IAM Role & Policy for Amplify
amplify_role = iam.Role(
self,
'amplify-wild-rydes-role',
role_name='amplify-wild-rydes-role',
assumed_by=iam.ServicePrincipal('amplify.amazonaws.com'))
# Amplify
amplify_static_site = amplify.App(
self,
'amplify-wild-rydes-site',
source_code_provider=amplify.CodeCommitSourceCodeProvider(
repository=amplify_repo),
description='Wild Rydes Amplify Static Site',
role=amplify_role,
app_name='wild-rydes-site')
master = amplify_static_site.add_branch("master")
# Policy is fairly open
# Ran into issues when I deployed the cognito user pools through the amplify cli
# It creates a new CloudFormation stack and deploys several resources
amplify_policy = iam.Policy(
self,
'amplify-wild-rydes-policy',
roles=[amplify_role],
statements=[
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=['codecommit:GitPull'],
resources=[amplify_repo.repository_arn]),
iam.PolicyStatement(effect=iam.Effect.ALLOW,
actions=[
'amplify:GetApp',
'amplify:CreateBackendEnvironment',
'cloudformation:*', 'cognito:*',
'lambda:*', 's3:*', 'iam:*'
],
resources=['*'])
])
# DynamoDB
# removal_policy=core.RemovalPolicy.DESTROY is to ensure it is deleted since this is only a lab
# table_name is required to be Rides, its configured in the nodejs code that the lambda function runs
rides_table = ddb.Table(self,
'Table',
table_name='Rides',
partition_key=ddb.Attribute(
name='RideId',
type=ddb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.DESTROY)
# Lambda Functions
request_unicorn_role = iam.Role(
self,
'RequestUnicornRole',
role_name='wild-rydes-lambda-role',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole')
])
# Grant write access to the lambda role
rides_table.grant_write_data(request_unicorn_role)
request_unicorn = _lambda.Function(
self,
'request-unicorn',
handler='requestUnicorn.handler',
runtime=_lambda.Runtime.NODEJS_12_X,
code=_lambda.AssetCode('request_unicorn'),
role=request_unicorn_role,
function_name='request-unicorn-wild-rydes')
# Rest API
ride_api_gw = apigw.RestApi(
self,
'wild-rydes-apigw',
rest_api_name='WildRydes',
endpoint_types=[apigw.EndpointType.REGIONAL])
# APIGW Lambda Integration
# proxy enabled for the workshop
ride_api_gw_lambda_integration = apigw.LambdaIntegration(
request_unicorn,
proxy=True,
integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin':
"'*'",
}
}])
post_ride_resource = ride_api_gw.root.add_resource('ride')
post_ride_resource_method = post_ride_resource.add_method(
'POST',
ride_api_gw_lambda_integration,
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Origin': True,
}
}])
# This needs to be created after the Amplify site unless you create the cognito userpool in the cdk
# I went through the Amplify CLI to create the backend
ride_api_gw_authorizer = apigw.CfnAuthorizer(
self,
'wild-rydes-apigw-authorizer',
rest_api_id=ride_api_gw.rest_api_id,
name='wild-rydes-apigw-authorizer',
type='COGNITO_USER_POOLS',
identity_source='method.request.header.name.Authorization',
identity_validation_expression="Bearer (.*)",
provider_arns=[
'arn:aws:cognito-idp:us-east-1:<ACCOUNT_ID>:userpool/<USER_POOL_ID>'
])
# https://github.com/aws/aws-cdk/issues/5618
post_ride_resource_fix = post_ride_resource_method.node.find_child(
'Resource')
post_ride_resource_fix.add_property_override('AuthorizationType',
'COGNITO_USER_POOLS')
post_ride_resource_fix.add_property_override(
'AuthorizerId', {"Ref": ride_api_gw_authorizer.logical_id})
# Enable CORS for the workshop
post_ride_resource.add_method(
'OPTIONS',
apigw.MockIntegration(integration_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Headers':
"'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'",
'method.response.header.Access-Control-Allow-Origin':
"'*'",
'method.response.header.Access-Control-Allow-Methods':
"'POST,OPTIONS'"
}
}],
passthrough_behavior=apigw.
PassthroughBehavior.WHEN_NO_MATCH,
request_templates={
"application/json":
"{\"statusCode\":200}"
}),
method_responses=[{
'statusCode': '200',
'responseParameters': {
'method.response.header.Access-Control-Allow-Headers':
True,
'method.response.header.Access-Control-Allow-Methods':
True,
'method.response.header.Access-Control-Allow-Origin': True,
}
}])
# Outputs
amplify_repo_url = core.CfnOutput(
self,
'amplify-repo-url',
value=amplify_repo.repository_clone_url_http)
app_repo_url = core.CfnOutput(self,
'app-repo-url',
value=app_repo.repository_clone_url_http)
amplify_default_domain = core.CfnOutput(
self,
'amplify-default-domain',
value=amplify_static_site.default_domain)
request_unicorn_apigw = core.CfnOutput(self,
'request-unicorn-apigw',
value=request_unicorn_apigw.url)
def __init__(self, scope: core.Construct, construct_id: str,
**kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
invoker_topic = sns.Topic(self, "experiment-invoker")
result_table = dynamodb.Table(
self,
id="result-table",
billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST,
partition_key=dynamodb.Attribute(
name="PK", type=dynamodb.AttributeType.STRING),
sort_key=dynamodb.Attribute(name="SK",
type=dynamodb.AttributeType.STRING),
removal_policy=core.RemovalPolicy.DESTROY,
)
lambda_code_asset = _lambda.Code.from_asset(
path=os.path.join(os.path.dirname(__file__), "..", "src"))
# Now we build as many lambdas as we need.
current_mem_size = LAMBDA_MEMORY_MIN_SIZE_IN_MB
while current_mem_size <= LAMBDA_MEMORY_MAX_SIZE_IN_MB:
# Build the function to test the client call
client_function = _lambda.Function(
self,
id=f"measurement-client-{current_mem_size}-mb",
code=lambda_code_asset,
environment={
"TEST_METHOD": "client",
"MEMORY_SIZE": str(current_mem_size),
"TABLE_NAME": result_table.table_name,
},
handler="lambda_handler.client_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
memory_size=current_mem_size)
client_function.add_event_source(
lambda_event_sources.SnsEventSource(invoker_topic))
result_table.grant_read_write_data(client_function)
# Allow for self-mutating function
client_function.add_to_role_policy(
iam.PolicyStatement(
actions=[
"lambda:getFunctionConfiguration",
"lambda:updateFunctionConfiguration",
],
# CFN screams at me with circular dependencies if I use the ref here.
resources=["*"]))
# Build the function to test the resource call
resource_function = _lambda.Function(
self,
id=f"measurement-resource-{current_mem_size}-mb",
code=lambda_code_asset,
environment={
"TEST_METHOD": "resource",
"MEMORY_SIZE": str(current_mem_size),
"TABLE_NAME": result_table.table_name
},
handler="lambda_handler.resource_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
memory_size=current_mem_size)
resource_function.add_event_source(
lambda_event_sources.SnsEventSource(invoker_topic))
result_table.grant_read_write_data(resource_function)
# Allow for self-mutating function
resource_function.add_to_role_policy(
iam.PolicyStatement(
actions=[
"lambda:getFunctionConfiguration",
"lambda:updateFunctionConfiguration",
],
# CFN screams at me with circular dependencies if I use the ref here.
resources=["*"]))
current_mem_size += LAMBDA_MEMORY_INCREMENTS_IN_MB
# The function to gather and aggregate the measurements
result_aggregator = _lambda.Function(
self,
id="result-aggregator",
code=lambda_code_asset,
environment={
"TABLE_NAME": result_table.table_name,
},
handler="lambda_handler.result_aggregator",
runtime=_lambda.Runtime.PYTHON_3_8,
memory_size=1024)
result_table.grant_read_write_data(result_aggregator)
# The function to gather and aggregate the measurements
invoker = _lambda.Function(self,
id="experiment-invoker-function",
code=lambda_code_asset,
environment={
"INVOKER_TOPIC_ARN":
invoker_topic.topic_arn,
},
handler="lambda_handler.invoke_handler",
runtime=_lambda.Runtime.PYTHON_3_8,
memory_size=1024)
invoker_topic.grant_publish(invoker)
core.CfnOutput(self,
"invokerFn",
value=invoker.function_name,
description="Name of the invoker function")
core.CfnOutput(self,
"resultAggregatorFn",
value=result_aggregator.function_name,
description="Name of the result aggregator function")
def __init__(self, scope: core.Construct, id: str,
artifact_bucket: s3.Bucket, static_website_bucket: s3.Bucket,
backend_fn: _lambda.Function, api: apigateway.LambdaRestApi,
**kwargs) -> None:
super().__init__(scope, id, **kwargs)
fn = _lambda.Function(
scope=self,
id="source-update-function",
runtime=_lambda.Runtime.PYTHON_3_8,
handler="index.handler",
# memory_size=500,
timeout=core.Duration.seconds(10),
code=_lambda.Code.from_asset(
os.path.join("lambdas", "updateSource")))
fn.add_to_role_policy(
statement=iam.PolicyStatement(actions=[
"lambda:UpdateFunctionCode",
],
resources=[
backend_fn.function_arn,
]))
fn.add_to_role_policy(
statement=iam.PolicyStatement(actions=[
"s3:GetObject",
],
resources=[
artifact_bucket.bucket_arn +
"/Server/main.zip",
]))
# Codepipeline
deploy_pipeline = codepipeline.Pipeline(
scope=self,
id="deploy-pipeline",
restart_execution_on_update=True,
)
lambda_source_output = codepipeline.Artifact()
client_source_output = codepipeline.Artifact()
deploy_pipeline.add_stage(stage_name="Source",
actions=[
codepipeline_actions.S3SourceAction(
action_name="LambdaSource",
bucket=artifact_bucket,
bucket_key="Server/main.zip",
output=lambda_source_output,
),
codepipeline_actions.S3SourceAction(
action_name="ClientSource",
bucket=artifact_bucket,
bucket_key="Client/src.zip",
output=client_source_output,
)
])
build_specs = {
"version": "0.2",
"env": {
"variables": {
"REACT_APP_AUTH_URL": api.url,
}
},
"phases": {
"install": {
"commands": [
"npm install -g yarn",
]
},
"build": {
"commands": [
"npm install",
"yarn test",
"yarn build",
]
}
},
"artifacts": {
"base-directory": "build",
"files": [
"**/*",
],
}
}
client_build_output = codepipeline.Artifact()
deploy_pipeline.add_stage(
stage_name="Build",
actions=[
codepipeline_actions.CodeBuildAction(
action_name="ClientBuild",
project=codebuild.Project(
scope=self,
id="codebuild-client",
build_spec=codebuild.BuildSpec.from_object(
build_specs),
),
input=client_source_output,
outputs=[client_build_output])
])
deploy_pipeline.add_stage(stage_name="Deploy",
actions=[
codepipeline_actions.LambdaInvokeAction(
lambda_=fn,
inputs=[lambda_source_output],
action_name="UpdateSource",
user_parameters={
"functionName":
backend_fn.function_name,
"sourceBucket":
artifact_bucket.bucket_name,
"sourceKey": "Server/main.zip",
}),
codepipeline_actions.S3DeployAction(
bucket=static_website_bucket,
input=client_build_output,
action_name="DeployClient",
extract=True,
),
])