def add_deployment(t: Template, rest_api: RestApi, lambda_methods):
# Create a deployment
stage_name = 'v1'
method_names = list(
map(lambda lambda_method: lambda_method.name, lambda_methods))
deployment = t.add_resource(
Deployment(
"%sDeployment" % stage_name,
DependsOn=method_names,
RestApiId=Ref(rest_api),
))
stage = t.add_resource(
Stage('%sStage' % stage_name,
StageName=stage_name,
RestApiId=Ref(rest_api),
DeploymentId=Ref(deployment)))
# Create an API usage plan
usagePlan = t.add_resource(
UsagePlan("ExampleUsagePlan",
UsagePlanName="ExampleUsagePlan",
Description="Example usage plan",
Quota=QuotaSettings(Limit=50000, Period="MONTH"),
Throttle=ThrottleSettings(BurstLimit=500, RateLimit=5000),
ApiStages=[ApiStage(ApiId=Ref(rest_api), Stage=Ref(stage))]))
# Add the deployment endpoint as an output
t.add_output([
Output("ApiEndpoint",
Value=Join("", [
"https://",
Ref(rest_api), ".execute-api.eu-west-1.amazonaws.com/",
stage_name
]),
Description="Endpoint for this stage of the api")
])
Type="AWS",
IntegrationHttpMethod='POST',
IntegrationResponses=[IntegrationResponse(StatusCode='200')],
Uri=Join("", [
"arn:aws:apigateway:eu-west-1:lambda:path/2015-03-31/functions/",
GetAtt("FoobarFunction", "Arn"), "/invocations"
])),
MethodResponses=[MethodResponse("CatResponse", StatusCode='200')]))
# Create a deployment
stage_name = 'v1'
deployment = t.add_resource(
Deployment(
"%sDeployment" % stage_name,
DependsOn="LambdaMethod",
RestApiId=Ref(rest_api),
))
stage = t.add_resource(
Stage('%sStage' % stage_name,
StageName=stage_name,
RestApiId=Ref(rest_api),
DeploymentId=Ref(deployment)))
key = t.add_resource(
ApiKey("ApiKey",
StageKeys=[StageKey(RestApiId=Ref(rest_api),
StageName=Ref(stage))]))
# Create an API usage plan
def __init__(self,
utils,
templatePath='./cloudformation/stage.json',
description='API Gateway Stage Template for {App}-{Stage}',
version='2010-09-09'):
super(self.__class__, self).__init__()
self.utils = utils
self.templatePath = templatePath
appName = self.utils.config['App']
stageName = self.utils.config['Stage']
tags = self.utils.config['Tags']
self.add_version(version)
self.add_description(description.format(App=appName, Stage=stageName))
####################
# Lambda Functions #
####################
self.lambdaFunctions = []
self.lambdaFunctionRoles = []
for functionId in self.utils.config['LambdaFunctions'].keys():
f = self.utils.config['LambdaFunctions'][functionId]
functionName = '{App}Stage{FunctionId}Function{Stage}'.format(
App=appName, FunctionId=functionId, Stage=stageName)
policyName = '{App}Stage{FunctionId}FunctionPolicy{Stage}'.format(
App=appName, FunctionId=functionId, Stage=stageName)
roleName = '{App}Stage{FunctionId}FunctionRole{Stage}'.format(
App=appName, FunctionId=functionId, Stage=stageName)
functionSubParams = {
'FunctionName': functionName,
'Api': ImportValue('{App}Api'.format(App=appName)),
'Stage': stageName.lower(),
}
#################
# Function Role #
#################
with open(
'./lambda/policies/{FunctionId}.json'.format(
FunctionId=functionId), 'r') as functionPolicyJson:
functionPolicyDocument = functionPolicyJson.read()
functionPolicy = Policy(
policyName,
PolicyName=policyName,
PolicyDocument=Sub(functionPolicyDocument,
**functionSubParams),
)
functionRole = self.add_resource(
Role(
roleName,
AssumeRolePolicyDocument=PolicyDocument(Statement=[
Statement(
Effect=Allow,
Action=[AssumeRole],
Principal=Principal('Service', [
'lambda.amazonaws.com',
]),
)
], ),
Path='/service-role/{App}/{Stage}/'.format(
App=appName, Stage=stageName),
Policies=[functionPolicy],
RoleName=roleName,
))
lambdaFunction = self.add_resource(
Function(
functionName,
Code=path.abspath(f['LocalCode']),
Description='API Proxy Function for Stage: {App}-{Stage}'.
format(App=appName, Stage=stageName),
FunctionName=functionName,
Handler=f['Handler'],
MemorySize=f['Memory'],
Role=GetAtt(functionRole, 'Arn'),
Runtime=f['Runtime'],
Timeout=f['Timeout'],
Tags=Tags(tags),
))
lambdaEnvironment = {
'APP_NAME': appName,
'STAGE_NAME': stageName,
}
if f['Environment'] != None:
lambdaEnvironment.update(f['Environment'])
lambdaFunction.Environment = Environment(
Variables=lambdaEnvironment, )
if f['KmsKeyArn'] != None:
lambdaFunction.KmsKeyArn = f['KmsKeyArn']
if f['Vpc'] != None:
lambdaFunction.VpcConfig = VpcConfig(
SecurityGroupIds=f['Vpc']['SecurityGroupIds'],
SubnetIds=f['Vpc']['SubnetIds'],
)
if f['Tracing'] != None:
lambdaFunction.TracingConfig = TracingConfig(
Mode=f['Tracing'], )
self.lambdaFunctions.append(lambdaFunction)
self.lambdaFunctionRoles.append(functionRole)
##################
# Lambda Proxies #
##################
self.proxyResources = []
self.proxyMethods = []
self.proxyMethodTitles = []
self.proxyPermissions = []
for resourceName in self.utils.config['LambdaProxies'].keys():
resource = self.utils.config['LambdaProxies'][resourceName]
resourcePath = resource['Path'].strip()
functionName = '{App}Stage{FunctionName}Function{Stage}'.format(
App=appName,
FunctionName=resource['Function'],
Stage=stageName)
resourceSubParams = {
'FunctionName': functionName,
'Api': ImportValue('{App}Api'.format(App=appName)),
'Stage': stageName.lower(),
}
if resourcePath == '':
proxyParent = ImportValue('{App}ApiRoot'.format(App=appName))
resourceSubParams['ResourcePath'] = '*'
else:
resourceSubParams['ResourcePath'] = resourcePath + '/*'
#################
# Path Resource #
#################
pathResource = self.add_resource(
Resource(
'{App}Stage{ResourceName}PathResource{Stage}'.format(
App=appName,
ResourceName=resourceName,
Stage=stageName),
ParentId=ImportValue(
'{App}ApiRoot'.format(App=appName)),
PathPart=resource['Path'],
RestApiId=ImportValue('{App}Api'.format(App=appName)),
))
self.proxyResources.append(pathResource)
proxyParent = Ref(pathResource)
###############
# Path Method #
###############
pathMethod = self.add_resource(
self.generate_proxy_method(
'{App}Stage{ResourceName}PathMethod{Stage}'.format(
App=appName,
ResourceName=resourceName,
Stage=stageName),
resource['Auth'],
resourceSubParams,
proxyParent,
ImportValue('{App}Api'.format(App=appName)),
))
self.proxyMethods.append(pathMethod)
self.proxyMethodTitles.append(pathMethod.title)
##################
# Proxy Resource #
##################
proxyResource = self.add_resource(
Resource(
'{App}Stage{ResourceName}ProxyResource{Stage}'.format(
App=appName,
ResourceName=resourceName,
Stage=stageName),
ParentId=proxyParent,
PathPart='{proxy+}',
RestApiId=ImportValue('{App}Api'.format(App=appName)),
))
self.proxyResources.append(proxyResource)
################
# Proxy Method #
################
proxyMethod = self.add_resource(
self.generate_proxy_method(
'{App}Stage{ResourceName}ProxyMethod{Stage}'.format(
App=appName,
ResourceName=resourceName,
Stage=stageName),
resource['Auth'],
resourceSubParams,
Ref(proxyResource),
ImportValue('{App}Api'.format(App=appName)),
))
self.proxyMethods.append(proxyMethod)
self.proxyMethodTitles.append(proxyMethod.title)
####################
# Proxy Permission #
####################
proxyPermission = self.add_resource(
Permission(
'{App}Stage{ResourceName}Permission{Stage}'.format(
App=appName,
ResourceName=resourceName,
Stage=stageName),
Action='lambda:InvokeFunction',
FunctionName=functionName,
Principal='apigateway.amazonaws.com',
SourceArn=Sub(
'arn:${AWS::Partition}:execute-api:${AWS::Region}:${AWS::AccountId}:${Api}/${Stage}/*/${ResourcePath}',
**resourceSubParams),
DependsOn=[functionName],
))
self.proxyPermissions.append(proxyPermission)
#################
# RestApi Stage #
#################
hashComponents = [self.proxyResources, self.proxyMethods]
deploymentHash = md5(pickle.dumps(hashComponents)).hexdigest()
self.deployment = self.add_resource(
Deployment(
'{App}StageDeployment{Stage}{Hash}'.format(
App=appName, Stage=stageName, Hash=deploymentHash),
Description='Deployment for {App} {Stage} Stage'.format(
App=appName, Stage=stageName),
RestApiId=ImportValue('{App}Api'.format(App=appName)),
DependsOn=self.proxyMethodTitles,
))
self.prodStage = self.add_resource(
Stage(
'{App}Stage'.format(App=appName),
DeploymentId=Ref(self.deployment),
Description='Stage for {App} {Stage} Stage.'.format(
App=appName, Stage=stageName, Run=self.utils.run_time),
MethodSettings=[
MethodSetting(
DataTraceEnabled=True,
HttpMethod='*',
LoggingLevel='INFO',
ResourcePath='/*',
#MetricsEnabled=True,
),
],
RestApiId=ImportValue('{App}Api'.format(App=appName)),
StageName=stageName.lower(),
))
##################
# Write Template #
##################
with open(templatePath, 'w') as templateFile:
templateFile.write(self.to_json())
Permission("UnsubscribePermission",
Action="lambda:InvokeFunction",
Principal="apigateway.amazonaws.com",
FunctionName=GetAtt(EmailUnsubscribeFunction, "Arn")))
resource = t.add_resource(
Permission("SendEmailPermission",
Action="lambda:InvokeFunction",
Principal="events.amazonaws.com",
FunctionName=GetAtt(EmailSendFunction, "Arn")))
# Deploy API
deployment = t.add_resource(
Deployment(
stage_name + "Deployment",
DependsOn=["subscribeMethod", "unsubscribeMethod"],
RestApiId=Ref(rest_api),
))
deployment_stage = t.add_resource(
Stage(stage_name + "Stage",
StageName=stage_name,
RestApiId=Ref(rest_api),
DeploymentId=Ref(deployment)))
t.add_output([
Output("ApiEndpoint",
Value=Join("", [
"https://",
Ref(rest_api), ".execute-api.",
Ref('AWS::Region'), ".amazonaws.com/", stage_name
def initiate_api_gateway_creation(self):
self.template.set_version('2010-09-09')
self.template.set_description('Creates a API Gateway which is '
'used to get data from dynamoDB.')
role = self.template.add_resource(
Role('RootRole',
RoleName="monty-cloud-api-role",
Path='/',
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": [
"apigateway.amazonaws.com",
"lambda.amazonaws.com",
"dynamodb.amazonaws.com"
]
}
}]
}))
self.template.add_resource(
ManagedPolicy(
'RolePolicies',
ManagedPolicyName='api-gw-policy',
Description='This policy is used for the DynamoDB table ',
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["dynamodb:*", "lambda:*", "s3:*"],
"Resource": [
"arn:aws:dynamodb:*:*:table/*",
"arn:aws:lambda:*:*:function:*"
],
"Effect":
"Allow"
}]
},
Roles=[Ref(role)]))
name = self.template.add_resource(
RestApi('restApiName',
Name='monty-cloud-get-api',
Description='Monty Cloud API Gateway',
EndpointConfiguration=EndpointConfiguration(
Types=['REGIONAL'])))
self.template.add_resource(
Permission("lambdaApiGatewayInvoke",
Action="lambda:InvokeFunction",
FunctionName="arn:aws:lambda:{}:{}:function:"
"get_data".format(self.region, self.account_number),
Principal="apigateway.amazonaws.com",
SourceArn="arn:aws:execute-api:{}:{}:*/*"
"/GET/get-details".format(self.region,
self.account_number)))
get_api_resource = self.template.add_resource(
Resource('restApiGetDetailsResource',
RestApiId=Ref(name),
ParentId=GetAtt(name, 'RootResourceId'),
PathPart='get-details',
DependsOn=name))
get_api_method = self.template.add_resource(
Method('restApiGetDetailsMethod',
AuthorizationType='None',
ApiKeyRequired=False,
HttpMethod='GET',
ResourceId=Ref(get_api_resource),
RestApiId=Ref(name),
Integration=Integration(Type='AWS_PROXY',
IntegrationHttpMethod='POST',
Uri=self.uri.format(
self.region, self.region,
self.account_number),
Credentials=GetAtt(role, "Arn")),
MethodResponses=[
MethodResponse(
StatusCode='200',
ResponseModels={'application/json': 'Empty'})
],
DependsOn=get_api_resource))
deployment = self.template.add_resource(
Deployment('restApiDeployment',
RestApiId=Ref(name),
DependsOn=[get_api_method]))
self.template.add_resource(
Stage('restApiStage',
DeploymentId=Ref(deployment),
Description='Prod Stage',
MethodSettings=[
MethodSetting(ResourcePath='/get-details',
HttpMethod='GET')
],
RestApiId=Ref(name),
StageName='prod'))
return self.template.to_yaml()
def add_api_gateway(self, apigateway_name):
self.log.info('Adding API Gateway %s' % apigateway_name)
assert (self.lambda_function is not None)
# define all value used by api gateway
lambda_method_name = '%sLambdaMethod' % apigateway_name
lambda_permission_name = '%sLambdaPermission' % apigateway_name
resource_name = '%sResource' % apigateway_name
deployment_name = '%sDeployment' % self.stage_name
apikey_name = '%sApiKey' % apigateway_name
# start creating api gateway template
self.apigateway = RestApi(apigateway_name, Name=apigateway_name)
self.template.add_resource(self.apigateway)
resource = Resource(resource_name,
RestApiId=Ref(self.apigateway),
PathPart='{proxy+}',
ParentId=GetAtt(apigateway_name, 'RootResourceId'))
self.template.add_resource(resource)
permission = Permission(lambda_permission_name,
Action='lambda:invokeFunction',
FunctionName=GetAtt(self.lambda_function,
'Arn'),
Principal='apigateway.amazonaws.com',
SourceArn=Join("", [
'arn:aws:execute-api:',
Ref('AWS::Region'), ':',
Ref('AWS::AccountId'), ':',
Ref(self.apigateway), '/*'
]))
self.template.add_resource(permission)
method = Method(
lambda_method_name,
DependsOn=lambda_permission_name,
RestApiId=Ref(self.apigateway),
ResourceId=Ref(resource),
HttpMethod='ANY',
AuthorizationType='NONE',
Integration=Integration(
Type='AWS_PROXY',
IntegrationHttpMethod='POST',
Uri=Join("", [
'arn:aws:apigateway:',
Ref('AWS::Region'), ':lambda:path/2015-03-31/functions/',
GetAtt(self.lambda_function, 'Arn'), '/invocations'
])),
MethodResponses=[MethodResponse(StatusCode='200')])
self.template.add_resource(method)
# create a deployment
deployment = Deployment(deployment_name,
DependsOn=lambda_method_name,
RestApiId=Ref(self.apigateway))
self.template.add_resource(deployment)
stage = Stage('%sStage' % self.stage_name,
StageName=self.stage_name,
RestApiId=Ref(self.apigateway),
DeploymentId=Ref(deployment))
self.template.add_resource(stage)
key = ApiKey(apikey_name,
StageKeys=[
StageKey(RestApiId=Ref(self.apigateway),
StageName=Ref(stage))
])
self.template.add_resource(key)
Integration=Integration(
Credentials=GetAtt("Civ6NotifLambdaExecutionRole", "Arn"),
Type="AWS",
IntegrationHttpMethod='POST',
IntegrationResponses=[IntegrationResponse(StatusCode='200')],
Uri=Join("", [
"arn:aws:apigateway:",
Ref("AWS::Region"), ":lambda:path/2015-03-31/functions/",
GetAtt("Civ6NotifFunction", "Arn"), "/invocations"
])),
MethodResponses=[MethodResponse("CatResponse", StatusCode='200')])
stage_name = "prod"
Civ6Notif_GW_Deployment = Deployment(f'{stage_name}Deployment',
RestApiId=Ref(Civ6Notif_GW),
DependsOn="Civ6NotifPostMethod")
Civ6Notif_GW_Stage = Stage(f'{stage_name}Stage',
StageName=stage_name,
RestApiId=Ref(Civ6Notif_GW),
DeploymentId=Ref(Civ6Notif_GW_Deployment))
Civ6Notif_Output_SNS = Output(
"SNSTopicArn",
Description="Arn of the SNS topic to subscribe to.",
Value=Join("", [
"arn:aws:sns:",
Ref("AWS::Region"), ":",
Ref("AWS::AccountId"), ":",
GetAtt("Civ6NotifTopic", "TopicName")
def register_resources_template(self, template):
deployment_resources = []
api = RestApi(
self.in_project_cf_name,
Name=troposphere.Join("-", [self.name, troposphere.Ref('Stage')]),
Description=self.settings.get('description', '')
)
template.add_resource(api)
deployment_resources.append(api)
invoke_lambda_role = troposphere.iam.Role(
utils.valid_cloudformation_name(self.name, 'Role'),
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Service": ["apigateway.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}]
},
Policies=[
troposphere.iam.Policy(
PolicyName=utils.valid_cloudformation_name(self.name, 'Role', 'Policy'),
PolicyDocument={
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction"
],
"Resource": [
"*"
]
}
]
}
)
]
)
template.add_resource(invoke_lambda_role)
deployment_resources.append(invoke_lambda_role)
deployment_dependencies = []
for path, resource in six.iteritems(self.settings.get('resources', {})):
resource_reference = self.get_or_create_resource(path, api, template)
methods = resource['methods']
if isinstance(methods, six.string_types):
methods = [methods]
if not isinstance(methods, dict):
method_properties = copy.deepcopy(resource)
method_properties.pop('methods', None)
methods = dict([[method, method_properties] for method in methods])
for method, configuration in six.iteritems(methods):
method_name = [self.name]
method_name.extend(path.split('/'))
method_name.append(method)
extra = {}
if 'parameters' in configuration:
extra['RequestParameters'] = configuration['parameters']
m = Method(
utils.valid_cloudformation_name(*method_name),
HttpMethod=method,
AuthorizationType=self.get_authorization_type(configuration),
ApiKeyRequired=self.get_api_key_required(configuration),
Integration=self.get_integration(configuration, invoke_lambda_role),
MethodResponses=self.get_method_responses(configuration),
ResourceId=resource_reference,
RestApiId=troposphere.Ref(api),
**extra
)
template.add_resource(m)
deployment_dependencies.append(m.name)
deployment_resources.append(m)
deploy_hash = hashlib.sha1(six.text_type(uuid.uuid4()).encode('utf-8')).hexdigest()
deploy = Deployment(
utils.valid_cloudformation_name(self.name, "Deployment", deploy_hash[:8]),
DependsOn=sorted(deployment_dependencies),
StageName=troposphere.Ref('Stage'),
RestApiId=troposphere.Ref(api)
)
template.add_resource(deploy)
if self._get_true_false('cli-output', 't'):
template.add_output([
troposphere.Output(
utils.valid_cloudformation_name("Clioutput", self.in_project_name),
Value=troposphere.Join(
"",
[
"https://",
troposphere.Ref(api),
".execute-api.",
troposphere.Ref(troposphere.AWS_REGION),
".amazonaws.com/",
troposphere.Ref('Stage')
]
),
)
])
def __init__(self, title, template, method_config):
"""
This class creates an API Gateway object with one or multiple methods attached.
AWS Cloud Formation Links:
RestApi: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-restapi.html
Resource: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-resource.html
Method: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-apigateway-method.html
Integration:
https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-apitgateway-method-integration.html
Troposhere link:
https://github.com/cloudtools/troposphere/blob/master/troposphere/apigateway.py
:param title: title of the api gateway and associated resources to be used in cloud formation
:param template: the troposphere template object to update
:param method_config: a list of one or many ApiGatewayMethodConfig objects with data prefilled from yaml values
values
"""
self.title = title
self.template = template
self.methods = []
self.method_responses = []
self.integration_responses = []
self.method_config = method_config
self.permissions = []
self.api = self.template.add_resource(
RestApi(self.title, Name=Join('-',
[Ref('AWS::StackName'), title])))
for method in self.method_config:
resource = self.create_resource(method)
self.get_responses(method)
integration = self.create_integration(
method, self.get_lambda_reference(method.lambda_unit))
self.add_method(resource, integration, method)
dependencies = []
for method in self.methods:
dependencies.append(method.title)
self.deployment = Deployment(
'{0}Deployment'.format(self.title),
Description=Join('', [
Ref('AWS::StackName'), ' Deployment created for APIGW ',
self.title
]),
RestApiId=Ref(self.api),
StageName='amz_deploy',
DependsOn=dependencies)
self.template.add_resource(self.deployment)
self.template.add_output(
Output(self.deployment.title + 'URL',
Description='URL of API deployment: {0}'.format(
self.deployment.title),
Value=Join('', [
'https://',
Ref(self.api), '.execute-api.',
Ref("AWS::Region"), '.amazonaws.com/',
self.deployment.StageName
])))
Ref("AWS::AccountId"),
":",
Ref(rest_api),
"/*/POST/convert"
])
))
#
# Deploy API Gateway
#
stage_name = 'dev'
deployment = t.add_resource(Deployment(
"Deployment",
DependsOn="MethodConvert",
RestApiId=Ref(rest_api),
))
stage = t.add_resource(Stage(
"Stage",
StageName=stage_name,
RestApiId=Ref(rest_api),
DeploymentId=Ref(deployment)
))
# add the deployment endpoint as an output
t.add_output([
Output(
"ApiEndpoint",
Value=Join("", [
def _build_resources(self):
self.template = Template()
self.template.set_version("2010-09-09")
self.template_initial = Template()
self.template.set_version("2010-09-09")
# S3 Bucket
s3_bucket_obj = Bucket("S3Bucket", AccessControl=Private)
s3_bucket = self.template.add_resource(s3_bucket_obj)
s3_bucket_output_res = Output("S3BucketName",
Value=Ref(s3_bucket),
Description="S3 bucket")
self.template.add_output(s3_bucket_output_res)
self.template_initial.add_resource(s3_bucket_obj)
self.template_initial.add_output(s3_bucket_output_res)
# Kinesis Firehose event_in compressor
event_compressor_name = self.build_resource_name("event-compressor")
event_compressor = DeliveryStream(
"EventCompressor",
DeliveryStreamName=event_compressor_name,
S3DestinationConfiguration=S3DestinationConfiguration(
BucketARN=GetAtt("S3Bucket", "Arn"),
BufferingHints=BufferingHints(IntervalInSeconds=60,
SizeInMBs=25),
# TODO
# CloudWatchLoggingOptions=CloudWatchLoggingOptions(
# Enabled=True, LogGroupName="FirehosEventCompressor", LogStreamName="FirehosEventCompressor",
# ),
CompressionFormat="GZIP",
Prefix=S3_ENRICHED_PREFIX,
RoleARN=GetAtt("LambdaExecutionRole", "Arn"),
),
)
self.template.add_resource(event_compressor)
# Lambda Execution Role
self.template.add_resource(
Role(
"LambdaExecutionRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*",
"Effect": "Allow"
},
{
"Action": ["lambda:*"],
"Resource": "*",
"Effect": "Allow"
},
{
"Action": ["s3:*"],
"Resource":
Join("",
[GetAtt("S3Bucket", "Arn"), "/*"]),
"Effect":
"Allow",
},
{
"Action": ["firehose:PutRecord"],
"Resource": "*",
"Effect": "Allow"
},
],
},
)
],
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": [
"lambda.amazonaws.com",
"apigateway.amazonaws.com",
"firehose.amazonaws.com",
]
},
}],
},
))
# Event Receiver Lambda
matomo_event_receiver_lambda_name = self.build_resource_name(
"matomo-event-receiver")
self.template.add_resource(
Function(
"LambdaMatomoEventReceiver",
FunctionName=matomo_event_receiver_lambda_name,
Code=Code(
S3Bucket=Ref(s3_bucket),
S3Key=
f"{S3_DEPLOYMENT_PREFIX}{self.artifact_filename_hashed(event_receiver_zip_path)}",
),
Handler="lambda.lambda_handler",
Environment=Environment(
Variables={
"S3_BUCKET":
Ref(s3_bucket),
"DELIVERY_STREAM_NAME":
event_compressor_name,
"IP_GEOCODING_ENABLED":
self.cfg.get("ip_geocoding_enabled"),
"IP_INFO_API_TOKEN":
self.cfg.get("ip_info_api_token"),
"USERSTACK_API_TOKEN":
self.cfg.get("userstack_api_token"),
"DEVICE_DETECTION_ENABLED":
self.cfg.get("device_detection_enabled"),
"IP_ADDRESS_MASKING_ENABLED":
self.cfg.get("ip_address_masking_enabled"),
}),
Role=GetAtt("LambdaExecutionRole", "Arn"),
Runtime="python3.7",
))
# API Gateway
api_gateway = self.template.add_resource(
RestApi("APIGateway",
Name=self.build_resource_name("api-gateway")))
# API Gateway Stage
api_gateway_deployment = self.template.add_resource(
Deployment(
f"APIGatewayDeployment{API_DEPLOYMENT_STAGE}",
DependsOn="APIGatewayLambdaMatomoEventReceiverMain",
RestApiId=Ref(api_gateway),
))
api_gateway_stage = self.template.add_resource(
Stage(
f"APIGatewayStage{API_DEPLOYMENT_STAGE}",
StageName=API_DEPLOYMENT_STAGE,
RestApiId=Ref(api_gateway),
DeploymentId=Ref(api_gateway_deployment),
))
# API Gateway usage plan
self.template.add_resource(
UsagePlan(
"APIGatewayUsagePlan",
UsagePlanName="APIGatewayUsagePlan",
Quota=QuotaSettings(Limit=50000, Period="MONTH"),
Throttle=ThrottleSettings(BurstLimit=500, RateLimit=5000),
ApiStages=[
ApiStage(ApiId=Ref(api_gateway),
Stage=Ref(api_gateway_stage))
],
))
# API Gateway resource to map the lambda function to
def _lambda_method_obj(resource, suffix):
resource = self.template.add_resource(resource)
return self.template.add_resource(
Method(
f"APIGatewayLambdaMatomoEventReceiver{suffix}",
DependsOn="LambdaMatomoEventReceiver",
RestApiId=Ref(api_gateway),
AuthorizationType="NONE",
ResourceId=Ref(resource),
HttpMethod="ANY",
Integration=Integration(
Credentials=GetAtt("LambdaExecutionRole", "Arn"),
Type="AWS_PROXY",
IntegrationHttpMethod="POST",
Uri=Join(
"",
[
f"arn:aws:apigateway:{self.region_name}:lambda:path/2015-03-31/functions/",
GetAtt("LambdaMatomoEventReceiver", "Arn"),
"/invocations",
],
),
),
), )
# API Gateway Lambda method
_lambda_method_obj(
Resource(
"APIGatewayResourceMatomoEventReceiverMain",
RestApiId=Ref(api_gateway),
PathPart="matomo-event-receiver",
ParentId=GetAtt("APIGateway", "RootResourceId"),
),
"Main",
)
# matomo.php path alias for the event receiver lambda
_lambda_method_obj(
Resource(
"APIGatewayResourceMatomoEventReceiverMatomo",
RestApiId=Ref(api_gateway),
PathPart="matomo.php",
ParentId=GetAtt("APIGateway", "RootResourceId"),
),
"Matomo",
)
self.template.add_output([
Output(
OUTPUT_API_GATEWAY_ENDPOINT,
Value=Join(
"",
[
"https://",
Ref(api_gateway),
f".execute-api.{self.region_name}.amazonaws.com/",
API_DEPLOYMENT_STAGE,
],
),
Description="API Endpoint",
),
Output("APIId", Value=Ref(api_gateway), Description="API ID"),
])
# Glue Execution Role
self.template.add_resource(
Role(
"GlueExecutionRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [
{
"Action": ["logs:*"],
"Resource": "arn:aws:logs:*:*:*",
"Effect": "Allow"
},
{
"Effect":
"Allow",
"Action": [
"glue:*",
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListAllMyBuckets",
"s3:GetBucketAcl",
"iam:ListRolePolicies",
"iam:GetRole",
"iam:GetRolePolicy",
"cloudwatch:PutMetricData",
],
"Resource": ["*"],
},
{
"Effect": "Allow",
"Action": ["s3:CreateBucket"],
"Resource": ["arn:aws:s3:::aws-glue-*"],
},
{
"Effect":
"Allow",
"Action": [
"s3:GetObject", "s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::aws-glue-*/*",
"arn:aws:s3:::*/*aws-glue-*/*"
],
},
{
"Effect":
"Allow",
"Action": ["s3:GetObject"],
"Resource": [
"arn:aws:s3:::crawler-public*",
"arn:aws:s3:::aws-glue-*"
],
},
{
"Effect":
"Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource":
["arn:aws:logs:*:*:/aws-glue/*"],
},
{
"Action": ["s3:*"],
"Resource":
Join("",
[GetAtt("S3Bucket", "Arn"), "/*"]),
"Effect":
"Allow",
},
{
"Action": ["iam:PassRole"],
"Effect":
"Allow",
"Resource": [
"arn:aws:iam::*:role/service-role/AWSGlueServiceRole*"
],
"Condition": {
"StringLike": {
"iam:PassedToService":
["glue.amazonaws.com"]
}
},
},
{
"Action": ["iam:PassRole"],
"Effect": "Allow",
"Resource":
"arn:aws:iam::*:role/AWSGlueServiceRole*",
"Condition": {
"StringLike": {
"iam:PassedToService":
["glue.amazonaws.com"]
}
},
},
],
},
)
],
AssumeRolePolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["glue.amazonaws.com"]
},
}],
},
))
# Glue Database
glue_catalog_id = Ref("AWS::AccountId")
glue_database_name = self.build_resource_name("").replace("-", "_")
glue_database = self.template.add_resource(
Database(
"GlueDatabase",
CatalogId=glue_catalog_id,
DatabaseInput=DatabaseInput(
Name=glue_database_name,
LocationUri=Join(
"",
["s3://",
Ref(s3_bucket), f"/{S3_TEPM_PREFIX}glue/"]),
),
))
# build enriched table schema
table_schema = []
table_fields = []
for field, data_type in event_schema.schema_to_glue_schema(
event_schema.ENRICHED):
table_schema.append(Column(Name=field, Type=data_type))
table_fields.append(field)
# Glue events enriched table
self.template.add_resource(
Table(
"GlueTableEventsEnriched",
DatabaseName=Ref(glue_database),
CatalogId=glue_catalog_id,
TableInput=TableInput(
Name="events_enriched",
TableType="EXTERNAL_TABLE",
StorageDescriptor=StorageDescriptor(
Columns=table_schema,
InputFormat="org.apache.hadoop.mapred.TextInputFormat",
OutputFormat=
"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location=Join(
"",
["s3://",
Ref(s3_bucket), "/", S3_ENRICHED_PREFIX]),
Compressed=True,
Parameters={
"classification": "json",
"compressionType": "gzip",
"typeOfData": "file"
},
SerdeInfo=SerdeInfo(
Parameters={"paths": ",".join(table_fields)},
SerializationLibrary=
"org.openx.data.jsonserde.JsonSerDe",
),
),
),
))
# add Name tag to all resources that supports tagging
for resource_name, resource in chain(
self.template_initial.resources.items(),
self.template.resources.items()):
if "Tags" not in resource.props:
continue
tags_to_add = Tags(
Name=f"{self.name}-{camel_case_to_dashed(resource_name)}")
tags_existing = getattr(resource, "Tags", Tags())
setattr(resource, "Tags", tags_existing + tags_to_add)
# add stack templates for enabled modules
if self.exists:
for module in self.modules:
echo.enum_elm(f"preparing stack for module {module.id}")
# add module prefix to outputs
module_stack = module.stack
outputs_prefixed = {}
for title, output in module_stack.outputs.items():
# e.g. emr-spark-cluster => EmrSparkCluster
module_name = dashed_to_camel_case(module.id)
output.title = f"{module_name}{output.title}"
outputs_prefixed[output.title] = output
module_stack.outputs = outputs_prefixed
# add Name attr to all resources that supports the Name attr
for resource_name, resource in module_stack.resources.items():
if "Name" not in resource.props:
continue
name = f"{self.name}-{module.id}-{camel_case_to_dashed(resource_name)}"
setattr(resource, "Name", name)
# add Name tag to all resources that supports tagging
for resource_name, resource in module_stack.resources.items():
if "Tags" not in resource.props:
continue
name_tag = f"{self.name}-{module.id}-{camel_case_to_dashed(resource_name)}"
tags_to_add = Tags(Name=name_tag)
tags_existing = getattr(resource, "Tags", Tags())
setattr(resource, "Tags", tags_existing + tags_to_add)
# deploy stack as nested stack
with TemporaryDirectory() as tmp_dir:
# write module stack tpl to tmp file
s3_resource = self.boto_session.resource("s3")
s3_bucket_name = self.get_output("S3BucketName")
tmp_file_path = Path(tmp_dir, f"{module.id}_stack_tpl.yml")
with io.open(tmp_file_path, "w+") as tmp_file_fh:
# upload nested stack tpl filt to s3
tmp_file_fh.write(module_stack.to_yaml())
tmp_file_fh.seek(0)
s3_filename = f"{S3_DEPLOYMENT_PREFIX}{self.artifact_filename_hashed(tmp_file_fh.name)}"
s3_resource.Object(
s3_bucket_name,
s3_filename).put(Body=tmp_file_fh.read())
# add stack resource
module_id = module.id
module_id = module_id.replace("-", "")
self.template.add_resource(
Stack(
module_id,
TemplateURL=
f"https://s3.amazonaws.com/{s3_bucket_name}/{s3_filename}",
))
Principal="apigateway.amazonaws.com",
SourceArn=Join("", [
"arn:aws:execute-api:",
Ref("AWS::Region"), ":",
Ref("AWS::AccountId"), ":",
Ref(rest_api), "/*/GET/CrimeData"
])))
# Create a deployment
stage_name = 'v1'
deployment = t.add_resource(
Deployment(
"%sDeployment" % stage_name,
DependsOn=[
"CrimeDataAPIMethod" + methods[0],
"CrimeDataAPIMethod" + methods[1]
],
RestApiId=Ref(rest_api),
))
stage = t.add_resource(
Stage('%sStage' % stage_name,
StageName=stage_name,
RestApiId=Ref(rest_api),
DeploymentId=Ref(deployment)))
# Add the deployment endpoint as an output
t.add_output([
Output(
"ApiEndpoint",
Value=Join("", [
StatusCode='200',
ResponseTemplates={'application/json': ''},
)
],
Uri=Join("", [
"arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/",
GetAtt("function", "Arn"), "/invocations"
]),
RequestTemplates={'application/json': '{"statusCode": 200}'},
),
))
deployment = t.add_resource(
Deployment(
"%sDeployment" % stage_name,
DependsOn="optionsmethod",
RestApiId=Ref(rest_api),
))
stage = t.add_resource(
Stage('%sStage' % stage_name,
StageName=stage_name,
RestApiId=Ref(rest_api),
DeploymentId=Ref(deployment)))
# Create cname record for all mount points
apiCname = t.add_resource(
RecordSetType(
'apiCname',
HostedZoneName='{}.'.format(dns_domain),
Comment="{} API gateway domain record".format(app_group_l),
"arn:aws:apigateway:", Ref(AWS_REGION), ":lambda:path/2015-03-31/functions/",
GetAtt(options_function, "Arn"),
"/invocations"
])
),
MethodResponses=[
MethodResponse(
"OptionsResponse",
StatusCode='200'
)
]
))
deployment = template.add_resource(Deployment(
"Deployment" + stage_name,
DependsOn=health_method,
RestApiId=Ref(api_gateway),
))
stage = template.add_resource(Stage(
'Stage' + stage_name,
StageName=stage_name,
RestApiId=Ref(api_gateway),
DeploymentId=Ref(deployment),
))
key = template.add_resource(ApiKey(
"ApiKey",
Enabled=True,
Value=api_key_secret,
StageKeys=[StageKey(
Integration=Integration(
Credentials=GetAtt("LambdaExecutionRole", "Arn"),
Type="AWS",
IntegrationHttpMethod='POST',
IntegrationResponses=[IntegrationResponse(StatusCode='200')],
Uri=Join("", [
"arn:aws:apigateway:eu-west-1:lambda:path/2015-03-31/functions/",
GetAtt("FoobarFunction", "Arn"), "/invocations"
])),
MethodResponses=[MethodResponse("CatResponse", StatusCode='200')]))
# Create a deployment
stage_name = 'v1'
deployment = t.add_resource(
Deployment("%sDeployment" % stage_name,
RestApiId=Ref(rest_api),
StageName=stage_name))
key = t.add_resource(
ApiKey("ApiKey",
StageKeys=[StageKey(RestApiId=Ref(rest_api), StageName=stage_name)],
Enabled=True))
# Add the deployment endpoint as an output
t.add_output([
Output(
"ApiEndpoint",
Value=Join("", [
"https://",
Ref(rest_api), ".execute-api.eu-west-1.amazonaws.com/", stage_name
]),
def get_framework_template(self):
from troposphere import (
GetAtt,
Ref,
Sub,
Tags,
Template,
)
from troposphere.apigateway import (
BasePathMapping,
Deployment,
DomainName,
Integration,
IntegrationResponse,
Method,
MethodResponse,
Resource,
RestApi,
Stage,
)
from troposphere.ec2 import (
VPCEndpoint, )
from troposphere.s3 import (
Bucket,
BucketPolicy,
VersioningConfiguration,
)
t = Template()
###############
# API Gateway #
###############
api = t.add_resource(
RestApi(
'ApiGateway',
Name=self.args.stack + 'Api',
Description=
'API for portal and redirects for the Cornell AppStream Service',
))
####################
# Redirect Methods #
####################
stack_url = "'https://shibidp.cit.cornell.edu/idp/profile/SAML2/Unsolicited/SSO?providerId=urn:amazon:webservices&target=https://appstream2.{region}.aws.amazon.com/saml?accountId={account}%26stack={stack}'"
stack_link = '<li><a href="./{redirect_nal}">{redirect}</a></li>'
methods = []
stack_links = ''
for redirect in sorted(self.config['Redirects'].keys()):
redirect_info = self.config['Redirects'][redirect]
redirect_nal = re.sub('\W+', '', redirect)
redirect_url = stack_url.format(account=redirect_info['account'],
region=redirect_info['region'],
stack=redirect_info['stack'])
methods.append('ApiGatewayRedirect' + redirect_nal)
stack_links += stack_link.format(redirect=redirect,
redirect_nal=redirect_nal)
resource = t.add_resource(
Resource(
'ApiGatewayResource' + redirect_nal,
ParentId=GetAtt(api, 'RootResourceId'),
PathPart=redirect_nal,
RestApiId=Ref(api),
))
method = t.add_resource(
Method(
'ApiGatewayRedirect' + redirect_nal,
AuthorizationType='None',
HttpMethod='ANY',
Integration=Integration(
Type='MOCK',
IntegrationResponses=[
IntegrationResponse(
ResponseParameters={
'method.response.header.Location':
redirect_url,
},
ResponseTemplates={
'application/json': '{"redirect": 302}'
},
StatusCode='302',
),
],
RequestTemplates={
'application/json': '{"statusCode":200}'
},
),
MethodResponses=[
MethodResponse(
ResponseParameters={
'method.response.header.Location': True,
},
StatusCode='302',
),
],
ResourceId=Ref(resource),
RestApiId=Ref(api),
))
###########################
# API Gateway Root Method #
###########################
with open('./include/root_integration_template.html',
'r') as rootTemplateHTML:
rootTemplate = rootTemplateHTML.read()
root_method = t.add_resource(
Method(
'ApiGatewayRootMethod',
AuthorizationType='None',
HttpMethod='ANY',
Integration=Integration(
Type='MOCK',
IntegrationResponses=[
IntegrationResponse(
ResponseParameters={
'method.response.header.Content-Type':
"'text/html'",
},
ResponseTemplates={
'text/html':
rootTemplate.format(stack_links=stack_links),
},
StatusCode='200',
),
],
RequestTemplates={
'application/json': '{"statusCode":200}'
},
),
MethodResponses=[
MethodResponse(
ResponseParameters={
'method.response.header.Content-Type': True,
},
StatusCode='200',
),
],
ResourceId=GetAtt(api, 'RootResourceId'),
RestApiId=Ref(api),
))
#####################
# API Gateway Stage #
#####################
api_deployment = t.add_resource(
Deployment(
'ApiGatewayDeployment' + self.run_time,
Description=
'Deployment for API portal and redirects for the Cornell AppStream Service',
RestApiId=Ref(api),
DependsOn=methods + ['ApiGatewayRootMethod'],
))
api_stage = t.add_resource(
Stage(
'ApiGatewayStage',
DeploymentId=Ref(api_deployment),
Description=
'Stage for API portal and redirects for the Cornell AppStream Service',
RestApiId=Ref(api),
StageName='apps',
))
######################
# API Gateway Domain #
######################
api_domain = t.add_resource(
DomainName(
'ApiGatewayDomain',
CertificateArn=self.config['ACM_ARN'],
DomainName=self.config['DomainName'],
))
api_domain_mapping = t.add_resource(
BasePathMapping(
'ApiGatewayDomainMapping',
DomainName=Ref(api_domain),
RestApiId=Ref(api),
Stage=Ref(api_stage),
))
###################
# VPC S3 Endpoint #
###################
s3_endpoint = t.add_resource(
VPCEndpoint(
'S3VPCEndpoint',
ServiceName=Sub('com.amazonaws.${AWS::Region}.s3'),
VpcId=self.config['VPC'],
RouteTableIds=self.config['RouteTables'],
))
####################
# S3 Bucket Policy #
####################
sub_args = {
'bucket_name': self.config['Bucket'],
'vpc_id': self.config['VPC']
}
with open('./include/bucket_policy.json', 'r') as bucketPolicyJSON:
bucket_policy_document = bucketPolicyJSON.read()
bucket_policy = t.add_resource(
BucketPolicy(
'FrameworkBucketPolicy',
Bucket=self.config['Bucket'],
PolicyDocument=Sub(bucket_policy_document, **sub_args),
))
with open('./cloudformation/framework.json', 'w') as frameworkTemplate:
frameworkTemplate.write(t.to_json())
return t