from troposphere import Ref, Template, Output
from troposphere.apigateway import RestApi, Method
from troposphere.apigateway import Resource, MethodResponse
from troposphere.apigateway import Integration, IntegrationResponse
from troposphere.apigateway import Deployment, Stage, ApiStage
from troposphere.apigateway import UsagePlan, QuotaSettings, ThrottleSettings
from troposphere.apigateway import ApiKey, StageKey, UsagePlanKey
from troposphere.iam import Role, Policy
from troposphere.awslambda import Function, Code
from troposphere import GetAtt, Join
t = Template()
# Create the Api Gateway
rest_api = t.add_resource(RestApi("ExampleApi", Name="ExampleApi"))
# Create a Lambda function that will be mapped
code = [
"var response = require('cfn-response');",
"exports.handler = function(event, context) {",
" context.succeed('foobar!');",
" return 'foobar!';",
"};",
]
# Create a role for the lambda function
t.add_resource(
Role(
"LambdaExecutionRole",
Path="/",
Policies=[
file.close()
EmailSendFunction = t.add_resource(
Function(
"EmailSendFunction",
Code=Code(ZipFile=Join("", code)),
Handler="index.daily_quote_handler",
Role=GetAtt(LambdaSesExecutionRole, "Arn"),
Runtime="python2.7",
Environment=Environment(
Variables={"email_from": Ref(from_email_param)}),
Description="Sends the Daily Quote. No endpoint, runs on a schedule.",
FunctionName="EmailSendFunction"))
### API GATEWAY
rest_api = t.add_resource(RestApi("EmailListApi", Name="EmailListApi"))
# /subscribe (POST)
subscribe_resource = t.add_resource(
Resource("subscribeResource",
RestApiId=Ref(rest_api),
PathPart="subscribe",
ParentId=GetAtt("EmailListApi", "RootResourceId")))
subscribe_method = t.add_resource(
Method("subscribeMethod",
DependsOn="EmailSubscribeFunction",
RestApiId=Ref(rest_api),
AuthorizationType="NONE",
ResourceId=Ref(subscribe_resource),
HttpMethod="POST",
def __init__(self,
utils,
templatePath='./cloudformation/api.json',
description='Top Level API Gateway Template for {App}',
version='2010-09-09'):
super(self.__class__, self).__init__()
self.utils = utils
self.templatePath = templatePath
appName = self.utils.config['App']
domainName = self.utils.config['Domain']
tags = self.utils.config['Tags']
self.add_version(version)
self.add_description(description.format(App=appName))
###################
# ACM Certificate #
###################
self.certificate = self.add_resource(
Certificate(
'{App}Certificate'.format(App=appName),
DomainName=domainName,
SubjectAlternativeNames=[
'*.{Domain}'.format(Domain=domainName)
],
Tags=Tags(tags),
))
#####################
# Deployment Bucket #
#####################
self.bucket = self.add_resource(
Bucket(
'{App}DeploymentBucket'.format(App=appName),
DeletionPolicy='Retain',
Tags=Tags(tags),
VersioningConfiguration=VersioningConfiguration(
Status='Enabled', ),
))
###########
# RestApi #
###########
self.api = self.add_resource(
RestApi(
'{App}Api'.format(App=appName),
Name=appName + 'Api',
Description='API for {App} AWS SAML Login Redirection'.format(
App=appName),
))
##################
# RestApi Domain #
##################
self.apiDomain = self.add_resource(
DomainName(
'{App}ApiDomainName'.format(App=appName),
CertificateArn=Ref(self.certificate),
DomainName=domainName,
))
self.apiDomainMapping = self.add_resource(
BasePathMapping(
'{App}ApiDomainNameMapping'.format(App=appName),
DomainName=Ref(self.apiDomain),
RestApiId=Ref(self.api),
))
###########
# Outputs #
###########
self.add_output(
Output(
'{App}Api'.format(App=appName),
Value=Ref(self.api),
Export=Export('{App}Api'.format(App=appName), ),
))
self.add_output(
Output(
'{App}ApiDomainName'.format(App=appName),
Value=Ref(self.apiDomain),
Export=Export('{App}ApiDomainName'.format(App=appName), ),
))
self.add_output(
Output(
'{App}ApiDomainDistribution'.format(App=appName),
Value=GetAtt(self.apiDomain, 'DistributionDomainName'),
Export=Export(
'{App}ApiDomainDistribution'.format(App=appName), ),
))
self.add_output(
Output(
'{App}ApiRoot'.format(App=appName),
Value=GetAtt(self.api, 'RootResourceId'),
Export=Export('{App}ApiRoot'.format(App=appName), ),
))
self.add_output(
Output(
'{App}Certificate'.format(App=appName),
Value=Ref(self.certificate),
Export=Export('{App}Certificate'.format(App=appName), ),
))
self.add_output(
Output(
'{App}DeploymentBucket'.format(App=appName),
Value=Ref(self.bucket),
Export=Export('{App}DeploymentBucket'.format(App=appName), ),
))
##################
# Write Template #
##################
with open(templatePath, 'w') as templateFile:
templateFile.write(self.to_json())
def add_api_gateway(self, apigateway_name):
self.log.info('Adding API Gateway %s' % apigateway_name)
assert (self.lambda_function is not None)
# define all value used by api gateway
lambda_method_name = '%sLambdaMethod' % apigateway_name
lambda_permission_name = '%sLambdaPermission' % apigateway_name
resource_name = '%sResource' % apigateway_name
deployment_name = '%sDeployment' % self.stage_name
apikey_name = '%sApiKey' % apigateway_name
# start creating api gateway template
self.apigateway = RestApi(apigateway_name, Name=apigateway_name)
self.template.add_resource(self.apigateway)
resource = Resource(resource_name,
RestApiId=Ref(self.apigateway),
PathPart='{proxy+}',
ParentId=GetAtt(apigateway_name, 'RootResourceId'))
self.template.add_resource(resource)
permission = Permission(lambda_permission_name,
Action='lambda:invokeFunction',
FunctionName=GetAtt(self.lambda_function,
'Arn'),
Principal='apigateway.amazonaws.com',
SourceArn=Join("", [
'arn:aws:execute-api:',
Ref('AWS::Region'), ':',
Ref('AWS::AccountId'), ':',
Ref(self.apigateway), '/*'
]))
self.template.add_resource(permission)
method = Method(
lambda_method_name,
DependsOn=lambda_permission_name,
RestApiId=Ref(self.apigateway),
ResourceId=Ref(resource),
HttpMethod='ANY',
AuthorizationType='NONE',
Integration=Integration(
Type='AWS_PROXY',
IntegrationHttpMethod='POST',
Uri=Join("", [
'arn:aws:apigateway:',
Ref('AWS::Region'), ':lambda:path/2015-03-31/functions/',
GetAtt(self.lambda_function, 'Arn'), '/invocations'
])),
MethodResponses=[MethodResponse(StatusCode='200')])
self.template.add_resource(method)
# create a deployment
deployment = Deployment(deployment_name,
DependsOn=lambda_method_name,
RestApiId=Ref(self.apigateway))
self.template.add_resource(deployment)
stage = Stage('%sStage' % self.stage_name,
StageName=self.stage_name,
RestApiId=Ref(self.apigateway),
DeploymentId=Ref(deployment))
self.template.add_resource(stage)
key = ApiKey(apikey_name,
StageKeys=[
StageKey(RestApiId=Ref(self.apigateway),
StageName=Ref(stage))
])
self.template.add_resource(key)
def initiate_api_gateway_creation(self):
self.template.set_version('2010-09-09')
self.template.set_description('Creates a API Gateway which is '
'used to get data from dynamoDB.')
role = self.template.add_resource(
Role('RootRole',
RoleName="monty-cloud-api-role",
Path='/',
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": [
"apigateway.amazonaws.com",
"lambda.amazonaws.com",
"dynamodb.amazonaws.com"
]
}
}]
}))
self.template.add_resource(
ManagedPolicy(
'RolePolicies',
ManagedPolicyName='api-gw-policy',
Description='This policy is used for the DynamoDB table ',
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["dynamodb:*", "lambda:*", "s3:*"],
"Resource": [
"arn:aws:dynamodb:*:*:table/*",
"arn:aws:lambda:*:*:function:*"
],
"Effect":
"Allow"
}]
},
Roles=[Ref(role)]))
name = self.template.add_resource(
RestApi('restApiName',
Name='monty-cloud-get-api',
Description='Monty Cloud API Gateway',
EndpointConfiguration=EndpointConfiguration(
Types=['REGIONAL'])))
self.template.add_resource(
Permission("lambdaApiGatewayInvoke",
Action="lambda:InvokeFunction",
FunctionName="arn:aws:lambda:{}:{}:function:"
"get_data".format(self.region, self.account_number),
Principal="apigateway.amazonaws.com",
SourceArn="arn:aws:execute-api:{}:{}:*/*"
"/GET/get-details".format(self.region,
self.account_number)))
get_api_resource = self.template.add_resource(
Resource('restApiGetDetailsResource',
RestApiId=Ref(name),
ParentId=GetAtt(name, 'RootResourceId'),
PathPart='get-details',
DependsOn=name))
get_api_method = self.template.add_resource(
Method('restApiGetDetailsMethod',
AuthorizationType='None',
ApiKeyRequired=False,
HttpMethod='GET',
ResourceId=Ref(get_api_resource),
RestApiId=Ref(name),
Integration=Integration(Type='AWS_PROXY',
IntegrationHttpMethod='POST',
Uri=self.uri.format(
self.region, self.region,
self.account_number),
Credentials=GetAtt(role, "Arn")),
MethodResponses=[
MethodResponse(
StatusCode='200',
ResponseModels={'application/json': 'Empty'})
],
DependsOn=get_api_resource))
deployment = self.template.add_resource(
Deployment('restApiDeployment',
RestApiId=Ref(name),
DependsOn=[get_api_method]))
self.template.add_resource(
Stage('restApiStage',
DeploymentId=Ref(deployment),
Description='Prod Stage',
MethodSettings=[
MethodSetting(ResourcePath='/get-details',
HttpMethod='GET')
],
RestApiId=Ref(name),
StageName='prod'))
return self.template.to_yaml()
def register_resources_template(self, template):
deployment_resources = []
api = RestApi(
self.in_project_cf_name,
Name=troposphere.Join("-", [self.name, troposphere.Ref('Stage')]),
Description=self.settings.get('description', '')
)
template.add_resource(api)
deployment_resources.append(api)
invoke_lambda_role = troposphere.iam.Role(
utils.valid_cloudformation_name(self.name, 'Role'),
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["apigateway.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
Policies=[
troposphere.iam.Policy(
PolicyName=utils.valid_cloudformation_name(self.name, 'Role', 'Policy'),
PolicyDocument={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction"
],
"Resource": [
"*"
]
}
]
}
)
]
)
template.add_resource(invoke_lambda_role)
deployment_resources.append(invoke_lambda_role)
deployment_dependencies = []
for path, resource in six.iteritems(self.settings.get('resources', {})):
resource_reference = self.get_or_create_resource(path, api, template)
methods = resource['methods']
if isinstance(methods, six.string_types):
methods = [methods]
if not isinstance(methods, dict):
method_properties = copy.deepcopy(resource)
method_properties.pop('methods', None)
methods = dict([[method, method_properties] for method in methods])
for method, configuration in six.iteritems(methods):
method_name = [self.name]
method_name.extend(path.split('/'))
method_name.append(method)
extra = {}
if 'parameters' in configuration:
extra['RequestParameters'] = configuration['parameters']
m = Method(
utils.valid_cloudformation_name(*method_name),
HttpMethod=method,
AuthorizationType=self.get_authorization_type(configuration),
ApiKeyRequired=self.get_api_key_required(configuration),
Integration=self.get_integration(configuration, invoke_lambda_role),
MethodResponses=self.get_method_responses(configuration),
ResourceId=resource_reference,
RestApiId=troposphere.Ref(api),
**extra
)
template.add_resource(m)
deployment_dependencies.append(m.name)
deployment_resources.append(m)
deploy_hash = hashlib.sha1(six.text_type(uuid.uuid4()).encode('utf-8')).hexdigest()
deploy = Deployment(
utils.valid_cloudformation_name(self.name, "Deployment", deploy_hash[:8]),
DependsOn=sorted(deployment_dependencies),
StageName=troposphere.Ref('Stage'),
RestApiId=troposphere.Ref(api)
)
template.add_resource(deploy)
if self._get_true_false('cli-output', 't'):
template.add_output([
troposphere.Output(
utils.valid_cloudformation_name("Clioutput", self.in_project_name),
Value=troposphere.Join(
"",
[
"https://",
troposphere.Ref(api),
".execute-api.",
troposphere.Ref(troposphere.AWS_REGION),
".amazonaws.com/",
troposphere.Ref('Stage')
]
),
)
])
"SendToDiscord",
Type="String",
AllowedValues=["True", "False"],
Default="False",
Description=
"Enabling this will tell the lambda function to send messages to a Discord Webhook URL.",
)
DiscordWebhookURL = Parameter(
"DiscordWebhookURL",
Type="String",
Description=
"Please paste here the URL that Discord provided to you when you created your server's Webhook.",
NoEcho=True)
Civ6Notif_GW = RestApi("Civ6NotifGW", Name="Civ6Notifications")
Civ6Notif_Role = Role(
"Civ6NotifLambdaExecutionRole",
Path="/",
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service":
["lambda.amazonaws.com", "apigateway.amazonaws.com"]
}
}]
Role=GetAtt("LambdaExecutionRole", "Arn"),
Runtime="nodejs12.x",
MemorySize=128,
Timeout=50,
Layers=[Ref(layer)]
))
#
# API Gateway
#
# create the Api Gateway
rest_api = t.add_resource(RestApi(
"ApiGateway",
Name="black-white-troposhpere-lambda-sharp",
BinaryMediaTypes=["*/*"], # very important
EndpointConfiguration=EndpointConfiguration(
Types=["REGIONAL"]
)
))
# Create a resource to map the lambda function to
resource = t.add_resource(Resource(
"ResourceConvert",
RestApiId=Ref(rest_api),
PathPart="convert",
ParentId=GetAtt("ApiGateway", "RootResourceId"),
))
# create a Lambda API method for the Lambda resource
method = t.add_resource(Method(
"MethodConvert",
def __init__(self, title, template, method_config):
"""
This class creates an API Gateway object with one or multiple methods attached.
AWS Cloud Formation Links:
RestApi: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-restapi.html
Resource: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-resource.html
Method: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-method.html
Integration:
https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-apitgateway-method-integration.html
Troposhere link:
https://github.com/cloudtools/troposphere/blob/master/troposphere/apigateway.py
:param title: title of the api gateway and associated resources to be used in cloud formation
:param template: the troposphere template object to update
:param method_config: a list of one or many ApiGatewayMethodConfig objects with data prefilled from yaml values
values
"""
self.title = title
self.template = template
self.methods = []
self.method_responses = []
self.integration_responses = []
self.method_config = method_config
self.permissions = []
self.api = self.template.add_resource(
RestApi(self.title, Name=Join('-',
[Ref('AWS::StackName'), title])))
for method in self.method_config:
resource = self.create_resource(method)
self.get_responses(method)
integration = self.create_integration(
method, self.get_lambda_reference(method.lambda_unit))
self.add_method(resource, integration, method)
dependencies = []
for method in self.methods:
dependencies.append(method.title)
self.deployment = Deployment(
'{0}Deployment'.format(self.title),
Description=Join('', [
Ref('AWS::StackName'), ' Deployment created for APIGW ',
self.title
]),
RestApiId=Ref(self.api),
StageName='amz_deploy',
DependsOn=dependencies)
self.template.add_resource(self.deployment)
self.template.add_output(
Output(self.deployment.title + 'URL',
Description='URL of API deployment: {0}'.format(
self.deployment.title),
Value=Join('', [
'https://',
Ref(self.api), '.execute-api.',
Ref("AWS::Region"), '.amazonaws.com/',
self.deployment.StageName
])))
from troposphere.apigateway import RestApi, Method
from troposphere.apigateway import Resource, MethodResponse
from troposphere.apigateway import Integration, IntegrationResponse
from troposphere.apigateway import Deployment, Stage, ApiStage
from troposphere.apigateway import UsagePlan, QuotaSettings, ThrottleSettings
from troposphere.apigateway import ApiKey, StageKey, UsagePlanKey, IntegrationResponse
from troposphere.iam import Role, Policy
from troposphere.awslambda import Function, Code, Permission
from troposphere import GetAtt, Join
bucket = "imjacobclark-artifacts"
t = Template()
# Create the Api Gateway
rest_api = t.add_resource(RestApi("CrimeDataAPI", Name="CrimeDataAPI"))
# Create a role for the lambda function
t.add_resource(
Role(
"CrimeDataExecutionRole",
Path="/",
Policies=[
Policy(PolicyName="root",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*",
"Effect": "Allow"
def _build_resources(self):
self.template = Template()
self.template.set_version("2010-09-09")
self.template_initial = Template()
self.template.set_version("2010-09-09")
# S3 Bucket
s3_bucket_obj = Bucket("S3Bucket", AccessControl=Private)
s3_bucket = self.template.add_resource(s3_bucket_obj)
s3_bucket_output_res = Output("S3BucketName",
Value=Ref(s3_bucket),
Description="S3 bucket")
self.template.add_output(s3_bucket_output_res)
self.template_initial.add_resource(s3_bucket_obj)
self.template_initial.add_output(s3_bucket_output_res)
# Kinesis Firehose event_in compressor
event_compressor_name = self.build_resource_name("event-compressor")
event_compressor = DeliveryStream(
"EventCompressor",
DeliveryStreamName=event_compressor_name,
S3DestinationConfiguration=S3DestinationConfiguration(
BucketARN=GetAtt("S3Bucket", "Arn"),
BufferingHints=BufferingHints(IntervalInSeconds=60,
SizeInMBs=25),
# TODO
# CloudWatchLoggingOptions=CloudWatchLoggingOptions(
# Enabled=True, LogGroupName="FirehosEventCompressor", LogStreamName="FirehosEventCompressor",
# ),
CompressionFormat="GZIP",
Prefix=S3_ENRICHED_PREFIX,
RoleARN=GetAtt("LambdaExecutionRole", "Arn"),
),
)
self.template.add_resource(event_compressor)
# Lambda Execution Role
self.template.add_resource(
Role(
"LambdaExecutionRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*",
"Effect": "Allow"
},
{
"Action": ["lambda:*"],
"Resource": "*",
"Effect": "Allow"
},
{
"Action": ["s3:*"],
"Resource":
Join("",
[GetAtt("S3Bucket", "Arn"), "/*"]),
"Effect":
"Allow",
},
{
"Action": ["firehose:PutRecord"],
"Resource": "*",
"Effect": "Allow"
},
],
},
)
],
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": [
"lambda.amazonaws.com",
"apigateway.amazonaws.com",
"firehose.amazonaws.com",
]
},
}],
},
))
# Event Receiver Lambda
matomo_event_receiver_lambda_name = self.build_resource_name(
"matomo-event-receiver")
self.template.add_resource(
Function(
"LambdaMatomoEventReceiver",
FunctionName=matomo_event_receiver_lambda_name,
Code=Code(
S3Bucket=Ref(s3_bucket),
S3Key=
f"{S3_DEPLOYMENT_PREFIX}{self.artifact_filename_hashed(event_receiver_zip_path)}",
),
Handler="lambda.lambda_handler",
Environment=Environment(
Variables={
"S3_BUCKET":
Ref(s3_bucket),
"DELIVERY_STREAM_NAME":
event_compressor_name,
"IP_GEOCODING_ENABLED":
self.cfg.get("ip_geocoding_enabled"),
"IP_INFO_API_TOKEN":
self.cfg.get("ip_info_api_token"),
"USERSTACK_API_TOKEN":
self.cfg.get("userstack_api_token"),
"DEVICE_DETECTION_ENABLED":
self.cfg.get("device_detection_enabled"),
"IP_ADDRESS_MASKING_ENABLED":
self.cfg.get("ip_address_masking_enabled"),
}),
Role=GetAtt("LambdaExecutionRole", "Arn"),
Runtime="python3.7",
))
# API Gateway
api_gateway = self.template.add_resource(
RestApi("APIGateway",
Name=self.build_resource_name("api-gateway")))
# API Gateway Stage
api_gateway_deployment = self.template.add_resource(
Deployment(
f"APIGatewayDeployment{API_DEPLOYMENT_STAGE}",
DependsOn="APIGatewayLambdaMatomoEventReceiverMain",
RestApiId=Ref(api_gateway),
))
api_gateway_stage = self.template.add_resource(
Stage(
f"APIGatewayStage{API_DEPLOYMENT_STAGE}",
StageName=API_DEPLOYMENT_STAGE,
RestApiId=Ref(api_gateway),
DeploymentId=Ref(api_gateway_deployment),
))
# API Gateway usage plan
self.template.add_resource(
UsagePlan(
"APIGatewayUsagePlan",
UsagePlanName="APIGatewayUsagePlan",
Quota=QuotaSettings(Limit=50000, Period="MONTH"),
Throttle=ThrottleSettings(BurstLimit=500, RateLimit=5000),
ApiStages=[
ApiStage(ApiId=Ref(api_gateway),
Stage=Ref(api_gateway_stage))
],
))
# API Gateway resource to map the lambda function to
def _lambda_method_obj(resource, suffix):
resource = self.template.add_resource(resource)
return self.template.add_resource(
Method(
f"APIGatewayLambdaMatomoEventReceiver{suffix}",
DependsOn="LambdaMatomoEventReceiver",
RestApiId=Ref(api_gateway),
AuthorizationType="NONE",
ResourceId=Ref(resource),
HttpMethod="ANY",
Integration=Integration(
Credentials=GetAtt("LambdaExecutionRole", "Arn"),
Type="AWS_PROXY",
IntegrationHttpMethod="POST",
Uri=Join(
"",
[
f"arn:aws:apigateway:{self.region_name}:lambda:path/2015-03-31/functions/",
GetAtt("LambdaMatomoEventReceiver", "Arn"),
"/invocations",
],
),
),
), )
# API Gateway Lambda method
_lambda_method_obj(
Resource(
"APIGatewayResourceMatomoEventReceiverMain",
RestApiId=Ref(api_gateway),
PathPart="matomo-event-receiver",
ParentId=GetAtt("APIGateway", "RootResourceId"),
),
"Main",
)
# matomo.php path alias for the event receiver lambda
_lambda_method_obj(
Resource(
"APIGatewayResourceMatomoEventReceiverMatomo",
RestApiId=Ref(api_gateway),
PathPart="matomo.php",
ParentId=GetAtt("APIGateway", "RootResourceId"),
),
"Matomo",
)
self.template.add_output([
Output(
OUTPUT_API_GATEWAY_ENDPOINT,
Value=Join(
"",
[
"https://",
Ref(api_gateway),
f".execute-api.{self.region_name}.amazonaws.com/",
API_DEPLOYMENT_STAGE,
],
),
Description="API Endpoint",
),
Output("APIId", Value=Ref(api_gateway), Description="API ID"),
])
# Glue Execution Role
self.template.add_resource(
Role(
"GlueExecutionRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*",
"Effect": "Allow"
},
{
"Effect":
"Allow",
"Action": [
"glue:*",
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListAllMyBuckets",
"s3:GetBucketAcl",
"iam:ListRolePolicies",
"iam:GetRole",
"iam:GetRolePolicy",
"cloudwatch:PutMetricData",
],
"Resource": ["*"],
},
{
"Effect": "Allow",
"Action": ["s3:CreateBucket"],
"Resource": ["arn:aws:s3:::aws-glue-*"],
},
{
"Effect":
"Allow",
"Action": [
"s3:GetObject", "s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::aws-glue-*/*",
"arn:aws:s3:::*/*aws-glue-*/*"
],
},
{
"Effect":
"Allow",
"Action": ["s3:GetObject"],
"Resource": [
"arn:aws:s3:::crawler-public*",
"arn:aws:s3:::aws-glue-*"
],
},
{
"Effect":
"Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource":
["arn:aws:logs:*:*:/aws-glue/*"],
},
{
"Action": ["s3:*"],
"Resource":
Join("",
[GetAtt("S3Bucket", "Arn"), "/*"]),
"Effect":
"Allow",
},
{
"Action": ["iam:PassRole"],
"Effect":
"Allow",
"Resource": [
"arn:aws:iam::*:role/service-role/AWSGlueServiceRole*"
],
"Condition": {
"StringLike": {
"iam:PassedToService":
["glue.amazonaws.com"]
}
},
},
{
"Action": ["iam:PassRole"],
"Effect": "Allow",
"Resource":
"arn:aws:iam::*:role/AWSGlueServiceRole*",
"Condition": {
"StringLike": {
"iam:PassedToService":
["glue.amazonaws.com"]
}
},
},
],
},
)
],
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["glue.amazonaws.com"]
},
}],
},
))
# Glue Database
glue_catalog_id = Ref("AWS::AccountId")
glue_database_name = self.build_resource_name("").replace("-", "_")
glue_database = self.template.add_resource(
Database(
"GlueDatabase",
CatalogId=glue_catalog_id,
DatabaseInput=DatabaseInput(
Name=glue_database_name,
LocationUri=Join(
"",
["s3://",
Ref(s3_bucket), f"/{S3_TEPM_PREFIX}glue/"]),
),
))
# build enriched table schema
table_schema = []
table_fields = []
for field, data_type in event_schema.schema_to_glue_schema(
event_schema.ENRICHED):
table_schema.append(Column(Name=field, Type=data_type))
table_fields.append(field)
# Glue events enriched table
self.template.add_resource(
Table(
"GlueTableEventsEnriched",
DatabaseName=Ref(glue_database),
CatalogId=glue_catalog_id,
TableInput=TableInput(
Name="events_enriched",
TableType="EXTERNAL_TABLE",
StorageDescriptor=StorageDescriptor(
Columns=table_schema,
InputFormat="org.apache.hadoop.mapred.TextInputFormat",
OutputFormat=
"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location=Join(
"",
["s3://",
Ref(s3_bucket), "/", S3_ENRICHED_PREFIX]),
Compressed=True,
Parameters={
"classification": "json",
"compressionType": "gzip",
"typeOfData": "file"
},
SerdeInfo=SerdeInfo(
Parameters={"paths": ",".join(table_fields)},
SerializationLibrary=
"org.openx.data.jsonserde.JsonSerDe",
),
),
),
))
# add Name tag to all resources that supports tagging
for resource_name, resource in chain(
self.template_initial.resources.items(),
self.template.resources.items()):
if "Tags" not in resource.props:
continue
tags_to_add = Tags(
Name=f"{self.name}-{camel_case_to_dashed(resource_name)}")
tags_existing = getattr(resource, "Tags", Tags())
setattr(resource, "Tags", tags_existing + tags_to_add)
# add stack templates for enabled modules
if self.exists:
for module in self.modules:
echo.enum_elm(f"preparing stack for module {module.id}")
# add module prefix to outputs
module_stack = module.stack
outputs_prefixed = {}
for title, output in module_stack.outputs.items():
# e.g. emr-spark-cluster => EmrSparkCluster
module_name = dashed_to_camel_case(module.id)
output.title = f"{module_name}{output.title}"
outputs_prefixed[output.title] = output
module_stack.outputs = outputs_prefixed
# add Name attr to all resources that supports the Name attr
for resource_name, resource in module_stack.resources.items():
if "Name" not in resource.props:
continue
name = f"{self.name}-{module.id}-{camel_case_to_dashed(resource_name)}"
setattr(resource, "Name", name)
# add Name tag to all resources that supports tagging
for resource_name, resource in module_stack.resources.items():
if "Tags" not in resource.props:
continue
name_tag = f"{self.name}-{module.id}-{camel_case_to_dashed(resource_name)}"
tags_to_add = Tags(Name=name_tag)
tags_existing = getattr(resource, "Tags", Tags())
setattr(resource, "Tags", tags_existing + tags_to_add)
# deploy stack as nested stack
with TemporaryDirectory() as tmp_dir:
# write module stack tpl to tmp file
s3_resource = self.boto_session.resource("s3")
s3_bucket_name = self.get_output("S3BucketName")
tmp_file_path = Path(tmp_dir, f"{module.id}_stack_tpl.yml")
with io.open(tmp_file_path, "w+") as tmp_file_fh:
# upload nested stack tpl filt to s3
tmp_file_fh.write(module_stack.to_yaml())
tmp_file_fh.seek(0)
s3_filename = f"{S3_DEPLOYMENT_PREFIX}{self.artifact_filename_hashed(tmp_file_fh.name)}"
s3_resource.Object(
s3_bucket_name,
s3_filename).put(Body=tmp_file_fh.read())
# add stack resource
module_id = module.id
module_id = module_id.replace("-", "")
self.template.add_resource(
Stack(
module_id,
TemplateURL=
f"https://s3.amazonaws.com/{s3_bucket_name}/{s3_filename}",
))
def add_apigateway_to_lambda(t: Template):
# Create the Api Gateway
rest_api = t.add_resource(RestApi("RecordsApi", Name="ExampleApi"))
return rest_api
"DomainName"))))
# Redirect outputs
t.add_output([
Output(
'CDNDomainOutput{}'.format(src_domain.replace('.', '0')),
Description="Domain for CDN",
Value=GetAtt(cdnDistribution, 'DomainName'),
)
])
#####################################################################################################################
# API Gateway
#####################################################################################################################
rest_api = t.add_resource(
RestApi("api", Name="{}-{}".format(env_l, app_group_l)))
#####################################################################################################################
# DynamoDB table
#####################################################################################################################
myDynamoDB = t.add_resource(
Table("myDynamoDBTable",
TableName='counters',
AttributeDefinitions=[
AttributeDefinition(AttributeName='website', AttributeType='S')
],
KeySchema=[KeySchema(AttributeName='website', KeyType='HASH')],
ProvisionedThroughput=ProvisionedThroughput(
ReadCapacityUnits=readunits, WriteCapacityUnits=writeunits)))
#####################################################################################################################
))
cloudfront_certificate = template.add_resource(Certificate(
"CloudFrontCertificate",
DomainName=Ref(domain_name),
DomainValidationOptions=[DomainValidationOption(
DomainName=Ref(domain_name),
ValidationDomain=ImportValue(Join('-', [Ref(dns_stack), 'HostedZoneName'])),
)],
ValidationMethod='DNS',
))
api_gateway = template.add_resource(RestApi(
'ApiGateway',
Name=Ref(AWS_STACK_NAME),
Description='REST API to handle requests that fall through lambda @ edge.',
EndpointConfiguration=EndpointConfiguration(
Types=['REGIONAL'],
)
))
health_resource = template.add_resource(Resource(
'HealthResource',
RestApiId=Ref(api_gateway),
PathPart="health",
ParentId=GetAtt(api_gateway, "RootResourceId"),
))
upvote_resource = template.add_resource(Resource(
'UpvoteResource',
RestApiId=Ref(api_gateway),
PathPart="upvote",
def get_framework_template(self):
from troposphere import (
GetAtt,
Ref,
Sub,
Tags,
Template,
)
from troposphere.apigateway import (
BasePathMapping,
Deployment,
DomainName,
Integration,
IntegrationResponse,
Method,
MethodResponse,
Resource,
RestApi,
Stage,
)
from troposphere.ec2 import (
VPCEndpoint, )
from troposphere.s3 import (
Bucket,
BucketPolicy,
VersioningConfiguration,
)
t = Template()
###############
# API Gateway #
###############
api = t.add_resource(
RestApi(
'ApiGateway',
Name=self.args.stack + 'Api',
Description=
'API for portal and redirects for the Cornell AppStream Service',
))
####################
# Redirect Methods #
####################
stack_url = "'https://shibidp.cit.cornell.edu/idp/profile/SAML2/Unsolicited/SSO?providerId=urn:amazon:webservices&target=https://appstream2.{region}.aws.amazon.com/saml?accountId={account}%26stack={stack}'"
stack_link = '<li><a href="./{redirect_nal}">{redirect}</a></li>'
methods = []
stack_links = ''
for redirect in sorted(self.config['Redirects'].keys()):
redirect_info = self.config['Redirects'][redirect]
redirect_nal = re.sub('\W+', '', redirect)
redirect_url = stack_url.format(account=redirect_info['account'],
region=redirect_info['region'],
stack=redirect_info['stack'])
methods.append('ApiGatewayRedirect' + redirect_nal)
stack_links += stack_link.format(redirect=redirect,
redirect_nal=redirect_nal)
resource = t.add_resource(
Resource(
'ApiGatewayResource' + redirect_nal,
ParentId=GetAtt(api, 'RootResourceId'),
PathPart=redirect_nal,
RestApiId=Ref(api),
))
method = t.add_resource(
Method(
'ApiGatewayRedirect' + redirect_nal,
AuthorizationType='None',
HttpMethod='ANY',
Integration=Integration(
Type='MOCK',
IntegrationResponses=[
IntegrationResponse(
ResponseParameters={
'method.response.header.Location':
redirect_url,
},
ResponseTemplates={
'application/json': '{"redirect": 302}'
},
StatusCode='302',
),
],
RequestTemplates={
'application/json': '{"statusCode":200}'
},
),
MethodResponses=[
MethodResponse(
ResponseParameters={
'method.response.header.Location': True,
},
StatusCode='302',
),
],
ResourceId=Ref(resource),
RestApiId=Ref(api),
))
###########################
# API Gateway Root Method #
###########################
with open('./include/root_integration_template.html',
'r') as rootTemplateHTML:
rootTemplate = rootTemplateHTML.read()
root_method = t.add_resource(
Method(
'ApiGatewayRootMethod',
AuthorizationType='None',
HttpMethod='ANY',
Integration=Integration(
Type='MOCK',
IntegrationResponses=[
IntegrationResponse(
ResponseParameters={
'method.response.header.Content-Type':
"'text/html'",
},
ResponseTemplates={
'text/html':
rootTemplate.format(stack_links=stack_links),
},
StatusCode='200',
),
],
RequestTemplates={
'application/json': '{"statusCode":200}'
},
),
MethodResponses=[
MethodResponse(
ResponseParameters={
'method.response.header.Content-Type': True,
},
StatusCode='200',
),
],
ResourceId=GetAtt(api, 'RootResourceId'),
RestApiId=Ref(api),
))
#####################
# API Gateway Stage #
#####################
api_deployment = t.add_resource(
Deployment(
'ApiGatewayDeployment' + self.run_time,
Description=
'Deployment for API portal and redirects for the Cornell AppStream Service',
RestApiId=Ref(api),
DependsOn=methods + ['ApiGatewayRootMethod'],
))
api_stage = t.add_resource(
Stage(
'ApiGatewayStage',
DeploymentId=Ref(api_deployment),
Description=
'Stage for API portal and redirects for the Cornell AppStream Service',
RestApiId=Ref(api),
StageName='apps',
))
######################
# API Gateway Domain #
######################
api_domain = t.add_resource(
DomainName(
'ApiGatewayDomain',
CertificateArn=self.config['ACM_ARN'],
DomainName=self.config['DomainName'],
))
api_domain_mapping = t.add_resource(
BasePathMapping(
'ApiGatewayDomainMapping',
DomainName=Ref(api_domain),
RestApiId=Ref(api),
Stage=Ref(api_stage),
))
###################
# VPC S3 Endpoint #
###################
s3_endpoint = t.add_resource(
VPCEndpoint(
'S3VPCEndpoint',
ServiceName=Sub('com.amazonaws.${AWS::Region}.s3'),
VpcId=self.config['VPC'],
RouteTableIds=self.config['RouteTables'],
))
####################
# S3 Bucket Policy #
####################
sub_args = {
'bucket_name': self.config['Bucket'],
'vpc_id': self.config['VPC']
}
with open('./include/bucket_policy.json', 'r') as bucketPolicyJSON:
bucket_policy_document = bucketPolicyJSON.read()
bucket_policy = t.add_resource(
BucketPolicy(
'FrameworkBucketPolicy',
Bucket=self.config['Bucket'],
PolicyDocument=Sub(bucket_policy_document, **sub_args),
))
with open('./cloudformation/framework.json', 'w') as frameworkTemplate:
frameworkTemplate.write(t.to_json())
return t
def __init__(self, lambda_names):
self.t = Template()
self.lambda_uris = []
lambda_iam_policy_arn = self.t.add_parameter(Parameter(
"LambdaIAMPolicyARN",
Description="ARN of the base IAM policy for Lambda functions",
Type="String"
))
apigw_stage_name = self.t.add_parameter(Parameter(
"APIGWStageName",
Description="Stage name for API Gateway deployment",
Type="String"
))
json_mapping_template = self.t.add_parameter(Parameter(
"MappingTemplate",
Description="Mapping template for request body, uri and body params, and stage variables",
Default=MAPPING_TEMPLATE,
Type="String"
))
for l in sorted(lambda_names):
self.add_lambda_uri_parameters(l)
apigw_role = self.t.add_resource(Role(
"APIGWRole",
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["apigateway.amazonaws.com"]
}
}]
},
Policies=[Policy(
PolicyName="APIGateway",
PolicyDocument={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["lambda:InvokeFunction", "iam:PassRole"],
"Resource": "*",
},
]
})
],
))
apigw = self.t.add_resource(RestApi(
"APIGW",
Body=self.get_swagger()
))
custom_apigw_deployment_role = self.t.add_resource(Role(
"CustomAPIGWDeploymentRole",
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
}
}]
},
ManagedPolicyArns=[
Ref(lambda_iam_policy_arn),
],
Policies=[Policy(
PolicyName="CreateDeployment",
PolicyDocument={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["apigateway:POST"],
"Resource": [
Join('', [
"arn:aws:apigateway:",
Ref('AWS::Region'),
"::/restapis/",
Ref(apigw),
"/deployments",
]),
],
},
]
})
],
))
self.custom_apigw_deployment_lambda = self.t.add_resource(Function(
f"CustomAPIGWDeploymentLambda",
FunctionName="CustomAPIGWDeploymentLambda",
Handler="index.handler",
Runtime="python3.6",
Role=GetAtt(custom_apigw_deployment_role, "Arn"),
Code=Code(
ZipFile=self.get_custom_apigw_deployment_code()
)
))
apigw_deployment = self.t.add_resource(CustomAPIGWDeployment(
"APIGWDeployment",
ServiceToken=GetAtt(self.custom_apigw_deployment_lambda, "Arn"),
RestApiId=Ref(apigw),
StageName=Ref(apigw_stage_name),
LambdaUris=self.lambda_uris
))
self.t.add_output(Output(
"URL",
Value=Join('', [
'https://',
Ref(apigw),
'.execute-api.',
Ref('AWS::Region'),
'.amazonaws.com/',
Ref(apigw_stage_name),
]),
Description="API Gateway stage's URL"
))