def add_topic_and_lambda(self, topic_name, lambda_meta_data):
"""Adds a SNS topic and SAM Function to the CloudFormation template
:param topic_name: (str) the SNS topic name
:param lambda_meta_data: (LambdaMetaData) an object specifying info about the lambda
to create
:return: a sns.Topic object
"""
topic = sns.Topic(
"{}Topic".format(topic_name),
TopicName=topic_name,
)
self.add_resource(topic)
self.add_resource(
serverless.Function(
"{}Lambda".format(lambda_meta_data.name),
Description=lambda_meta_data.description,
MemorySize=128,
FunctionName=lambda_meta_data.name,
Runtime=LAMBDA_RUNTIME,
Handler='index.handler',
CodeUri='lambda-src/',
Timeout=10,
Environment=awslambda.Environment(
Variables={
'WEBHOOK_URL': lambda_meta_data.webhook_url,
'MESSAGE_PREFIX': lambda_meta_data.message_prefix,
}),
Events={
'SNS': serverless.SNSEvent('sns', Topic=Ref(topic)),
},
))
return topic
def setup(self, key, name):
import_name = key['ImportName'] if 'ImportName' in key else name
if 'Code' not in key:
self.Code = lbd.Code()
try:
self.Code.ZipFile = Join('', import_lambda(import_name))
except:
self.Code.ZipFile = (
'print("Use Code parameter in yaml '
f'or create file lib/lambas/{import_name}.code '
'with lambda code to execute.")')
auto_get_props(self, key, recurse=True)
self.FunctionName = Sub('${AWS::StackName}-${EnvRole}-%s' % name)
if 'Handler' not in key:
self.Handler = 'index.lambda_handler'
self.Role = GetAtt(f'RoleLambda{name}', 'Arn')
if all(k in key for k in ['SecurityGroupIds', 'SubnetIds']):
self.VpcConfig = lbd.VPCConfig('')
auto_get_props(self.VpcConfig, key, mapname=self.title)
# Variables - skip if atEdge - always set Env, EnvRole
try:
key['AtEdge']
except Exception as e:
self.Environment = lbd.Environment(Variables={
'Env': Ref('EnvShort'),
'EnvRole': Ref('EnvRole'),
})
if 'Variables' in key:
self.Environment.Variables.update({
varname: get_endvalue(f'{self.title}Variables{varname}')
for varname in key['Variables']
})
def showeach_function(showeach_lambda_role, lambda_environment_dict, s3_bucket,
s3_key_value, showeach_s3_versionid_value):
return awslambda.Function(
'ShoweachFunction',
Code=awslambda.Code(S3Bucket=Ref(s3_bucket),
S3Key=Join('', [s3_key_value, '/showeach.zip']),
S3ObjectVersion=showeach_s3_versionid_value),
Environment=awslambda.Environment(Variables=lambda_environment_dict),
Handler='showeach.lambda_handler',
Role=GetAtt(showeach_lambda_role, 'Arn'),
Runtime='python3.6',
Timeout=60 * 5)
def csvimport_function(csvimport_lambda_role, lambda_environment_dict,
s3_bucket, s3_key_value, csvimport_s3_versionid_value):
return awslambda.Function(
'CsvimportFunction',
Code=awslambda.Code(S3Bucket=Ref(s3_bucket),
S3Key=Join('', [s3_key_value, '/csvimport.zip']),
S3ObjectVersion=csvimport_s3_versionid_value),
Environment=awslambda.Environment(Variables=lambda_environment_dict),
Handler='csvimport.lambda_handler',
Role=GetAtt(csvimport_lambda_role, 'Arn'),
Runtime='python3.6',
Timeout=60 * 5)
def show_function(show_lambda_role, lambda_environment_dict, showeach_topic,
s3_bucket, s3_key_value, show_s3_versionid_value):
env = {
'SHOWEACH_TOPIC': Ref(showeach_topic),
}
env.update(lambda_environment_dict)
return awslambda.Function('ShowFunction',
Code=awslambda.Code(
S3Bucket=Ref(s3_bucket),
S3Key=Join('', [s3_key_value, '/show.zip']),
S3ObjectVersion=show_s3_versionid_value),
Environment=awslambda.Environment(Variables=env),
Handler='show.lambda_handler',
Role=GetAtt(show_lambda_role, 'Arn'),
Runtime='python3.6',
Timeout=60 * 5)
def archive_function(archive_lambda_role, lambda_environment_dict,
archiveeach_topic, s3_bucket, s3_key_value,
archive_s3_versionid_value):
env = {
'ARCHIVEEACH_TOPIC': Ref(archiveeach_topic),
}
env.update(lambda_environment_dict)
return awslambda.Function(
'ArchiveFunction',
Code=awslambda.Code(S3Bucket=Ref(s3_bucket),
S3Key=Join('', [s3_key_value, '/archive.zip']),
S3ObjectVersion=archive_s3_versionid_value),
Environment=awslambda.Environment(Variables=env),
Handler='archive.lambda_handler',
Role=GetAtt(archive_lambda_role, 'Arn'),
Runtime='python3.6',
Timeout=60 * 5)
def _create_lambda_function(self, code_property, task_function, role,
runtime):
# type: (awslambda.Code, TaskFunction, iam.Role, str) -> None
# TODO add support for versioning
function_handler = self._get_function_handler_string(
task_function.func)
title = self._get_function_logical_id(function_handler)
function_kwargs = {
"Code":
code_property,
"Handler":
function_handler,
"Role":
GetAtt(role, "Arn"),
"Runtime":
runtime,
"Environment":
awslambda.Environment(
Variables=task_function.environment_variables),
"Tags":
troposphere.Tags(task_function.tags),
}
if self._app.kms_key_arn is not None and len(
self._app.kms_key_arn) > 0:
function_kwargs["KmsKeyArn"] = self._app.kms_key_arn
if len(self._app.subnet_ids) > 0 or len(
self._app.security_group_ids) > 0:
function_kwargs["VpcConfig"] = awslambda.VPCConfig(
SubnetIds=self._app.subnet_ids,
SecurityGroupIds=self._app.security_group_ids)
if task_function.memory is not None:
function_kwargs["MemorySize"] = task_function.memory
if task_function.timeout is not None:
function_kwargs["Timeout"] = task_function.timeout
if task_function.activate_tracing:
function_kwargs["TracingConfig"] = awslambda.TracingConfig(
Mode="Active")
# TODO specify the function name? Maybe we don't care?
return awslambda.Function(title, **function_kwargs)
def create_template(self):
t = self.template
v = self.get_variables()
lambda_role = self.add_lambda_role(
'Ec2Dns', [('ec2', 'DescribeInstances'), ('ec2', 'DescribeTags'),
('route53', 'ChangeResourceRecordSets'),
('route53', 'ListResourceRecordSets'),
('route53', 'GetChange')])
func = t.add_resource(
awslambda.Function(
'Ec2DnsLambdaFunction',
Code=v['Code'],
Handler='ec2_route53_lambdas.ec2_dns.handler',
Role=GetAtt(lambda_role, 'Arn'),
Runtime='python2.7',
MemorySize=Ref('MemorySize'),
Timeout=Ref('Timeout'),
Environment=awslambda.Environment(
Variables={
'EC2_DNS_HOSTED_ZONE_ID': Ref('HostedZoneID'),
'EC2_DNS_VPC_IDS': Join(',', Ref('TargetVPCIDs')),
'EC2_DNS_VPC_DOMAINS': Join(',', Ref('TargetDomains')),
'EC2_DNS_RECORD_TTL': Ref('RecordTTL')
})))
self.add_lambda_events_rule(
'Ec2DnsInstanceChange',
func,
EventPattern={
'source': ['aws.ec2'],
'detail-type': ['EC2 Instance State-change Notification'],
"detail": {
"state": ["running", "stopped", "terminated"]
}
})
self.add_lambda_events_rule('Ec2DnsRefresh',
func,
ScheduleExpression=Ref('Schedule'))
])),
]))
lambda_function = t.add_resource(
awslambda.Function(
'LambdaFunction',
FunctionName=lambda_function_name,
Description='Extract zip file contents to S3',
Code=awslambda.Code(ZipFile=function_code),
Handler='index.lambda_handler',
Role=GetAtt(lambda_execution_role, 'Arn'),
Runtime='python2.7',
MemorySize='512',
Timeout='120',
Environment=awslambda.Environment(Variables={
'DST_BUCKET': Ref(dst_bucket),
# 'DST_PREFIX': Ref(param_dst_prefix)
}),
))
updater_lambda_permission = t.add_resource(
awslambda.Permission(
'LambdaPermission',
FunctionName=Ref(lambda_function),
Action='lambda:InvokeFunction',
Principal='s3.amazonaws.com',
SourceAccount=Sub('${AWS::AccountId}'),
SourceArn=Join('', ['arn:aws:s3:::', Ref(src_bucket)]),
))
#
# Output
iam.Role(
"FindNewExecutionRole",
Path="/",
Policies=[logs_policy, prev_time_s3_policy, lambda_invoke],
AssumeRolePolicyDocument=utils.get_static_policy('lambda-policy-doc'),
))
find_new_granules = t.add_resource(
utils.make_lambda_function(
name="find_new_granules",
role=lambda_exe_role,
lambda_params={
"Environment":
awslambda.Environment(
Variables={
'PREVIOUS_TIME_SSM_PARAM_NAME': Ref(ssm_previous_time),
'SCHEDULER_LAMBDA_NAME': Ref(hyp3_scheduler.scheduler)
}),
"MemorySize":
128,
"Timeout":
300
}))
find_new_target = events.Target("FindNewTarget",
Arn=GetAtt(find_new_granules, 'Arn'),
Id="FindNewFunction1")
find_new_event_rule = t.add_resource(
events.Rule("FindNewGranulesSchedule",
ScheduleExpression="rate(1 minute)",
'Runtime':
'python3.6',
'CodeUri':
serverless.S3Location('unused',
Bucket=Ref(param_s3_bucket_name),
Key=Ref(param_s3_key)),
'Environment':
awslambda.Environment(
Variables={
"COGNITO_USER_POOL_ID":
Ref(cognito_user_pool),
"COGNITO_DOMAIN_PREFIX":
GetAtt(cognito_user_pool_domain, 'Domain'),
"COGNITO_CLIENT_ID":
Ref(cognito_user_pool_client),
"COGNITO_CLIENT_SECRET":
GetAtt(cognito_user_pool_client, 'ClientSecret'),
"DOMAIN_NAME":
Join('.', [Ref(param_label),
Ref(param_hosted_zone_name)]),
"MAGIC_PATH":
magic_path,
"CONFIG_BUCKET":
Ref(config_bucket),
}),
'Role':
GetAtt(lambda_role, 'Arn'),
}
template.add_resource(
serverless.Function(
"Index",
def build_template(self):
deploy_alias = False
t = self._init_template()
self.jinja_env_vars()
role_param = t.add_parameter(
Parameter(self.role.output_role_arn(), Type='String'))
bucket_ref = t.add_parameter(
Parameter(self.s3_bucket.output_bucket_name(), Type='String'))
if self._deploying:
if not self.uploaded and self._bucket_name():
deploy_alias = True
self._determine_code_versions()
logger.info("S3 Key: {}".format(self.zip_name))
func = t.add_resource(
awslambda.Function(
'{}Function'.format(self.get_stack_name()),
FunctionName=self.get_stack_name(),
Handler=self.handler,
MemorySize=self.memory,
Timeout=self.timeout,
Runtime=self.runtime,
Role=Ref(role_param),
Environment=awslambda.Environment(Variables=self.vars),
Code=awslambda.Code(S3Bucket=Ref(bucket_ref),
S3Key=self.zip_name)))
if self.s3_version:
func.Code.S3ObjectVersion = self.s3_version
# vpc mode
if self.vpc_stack is not None:
if self.public_subnet:
subnets = self.vpc_stack.output_public_subnets()
else:
subnets = self.vpc_stack.output_private_subnets()
subnet_refs = [
Ref(utils.ensure_param(t, val, 'String')) for val in subnets
]
func.VpcConfig = awslambda.VPCConfig(SubnetIds=subnet_refs,
SecurityGroupIds=[])
for sg in self.security_groups:
sg_ref = Ref(
utils.ensure_param(t, sg.output_security_group(),
'String'))
func.VpcConfig.SecurityGroupIds.append(sg_ref)
if deploy_alias is True:
for v in self.aliases:
t.add_resource(
awslambda.Alias('{}Alias'.format(v['name']),
FunctionName=Ref(func),
Name=v['name'],
FunctionVersion=v['version']))
if len(self.event_sources) > 0:
for s in self.event_sources:
src = s['src']
args = s['args']
if isinstance(src, dynamodb.DynamoTable):
p = t.add_parameter(
Parameter(src.output_stream(), Type='String'))
t.add_resource(
awslambda.EventSourceMapping(
'LambdaDynamo{}'.format(src.name),
FunctionName=Ref(func),
EventSourceArn=Ref(p),
StartingPosition='LATEST'))
if isinstance(src, sqs.Queue):
p = t.add_parameter(
Parameter(src.output_queue_arn(), Type='String'))
t.add_resource(
awslambda.EventSourceMapping(
'LambdaSQS{}'.format(src.name),
FunctionName=Ref(func),
EventSourceArn=Ref(p),
BatchSize=args.get('BatchSize', 1)))
for k, v in enumerate(self.perms):
v.build(t, funv, k)
t.add_output([
Output('FunctionName', Value=Ref(func)),
Output('FunctionArn', Value=GetAtt(func, "Arn"))
])
return t
awslambda.Function(
"datadogmonitorlambda",
DependsOn=["LogGroup"], # log_group.title would also work
Code=awslambda.Code(S3Bucket=Select(0, Ref(lambda_package)),
S3Key=Select(1, Ref(lambda_package))),
Handler="index.handler",
FunctionName=Join("-", ["datadogmonitorlambda",
Ref("AWS::StackName")]),
Role=GetAtt(datadog_lambda_role, "Arn"),
Runtime="python2.7",
Timeout=300,
MemorySize=1536,
KmsKeyArn=kms_key_arn,
Environment=awslambda.Environment(
Variables={
'api_key': GetAtt(api_key, "CiphertextBase64"),
'application_key': GetAtt(application_key, "CiphertextBase64"),
"LOG_LEVEL": Ref(log_level)
})))
datadog_timeboard_lambda = t.add_resource(
awslambda.Function(
"datadogtimeboardlambda",
DependsOn=["LogGroup"], # log_group.title would also work
Code=awslambda.Code(S3Bucket=Select(0, Ref(lambda_package)),
S3Key=Select(1, Ref(lambda_package))),
Handler="timeboard_index.handler",
FunctionName=Join("-",
["datadogtimeboardlambda",
Ref("AWS::StackName")]),
Role=GetAtt(datadog_lambda_role, "Arn"),
Runtime="python2.7",
def register_resources_template(self, template):
"""Register the lambda Function into the troposphere template. If
this function requires a custom Role, register it too."""
role = self.get_role()
env = self.get_environment()
depends_on = []
if isinstance(role, iam.Role):
template.add_resource(role)
depends_on.append(role.name)
role = troposphere.GetAtt(role, 'Arn')
template.add_parameter(
troposphere.Parameter(
utils.valid_cloudformation_name(self.name, "s3version"),
Type="String",
))
extra = {}
if self.settings.get('vpc'):
vpc = self.project.get_resource('vpc::{}'.format(
self.settings.get('vpc')))
if isinstance(vpc.settings['security-groups'], troposphere.Ref):
vpc.settings[
'security-groups']._type = 'List<AWS::EC2::SecurityGroup::Id>'
if isinstance(vpc.settings['subnet-ids'], troposphere.Ref):
vpc.settings['subnet-ids']._type = 'List<AWS::EC2::Subnet::Id>'
extra['VpcConfig'] = awslambda.VPCConfig(
SecurityGroupIds=vpc.settings['security-groups'],
SubnetIds=vpc.settings['subnet-ids'])
function = template.add_resource(
awslambda.Function(
self.in_project_cf_name,
DependsOn=depends_on,
Code=awslambda.Code(
S3Bucket=troposphere.Ref("CodeBucket"),
S3Key=self.get_bucket_key(),
S3ObjectVersion=troposphere.Ref(
utils.valid_cloudformation_name(
self.name, "s3version")),
),
Description=self.settings.get('description', ''),
Handler=self.get_handler(),
MemorySize=self.get_memory(),
Role=role,
Runtime=self.get_runtime(),
Timeout=self.get_timeout(),
Environment=awslambda.Environment(Variables=env),
**extra))
lambda_version = 'lambda:contrib_lambdas:version'
lambda_ref = troposphere.GetAtt(self.project.reference(lambda_version),
'Arn')
if not self.in_project_name.startswith('lambda:contrib_lambdas:'):
lambda_version = '{}:current'.format(lambda_version)
lambda_ref = troposphere.Ref(
self.project.reference(lambda_version))
version = template.add_resource(
LambdaVersion.create_with(
utils.valid_cloudformation_name(self.name, "Version"),
DependsOn=[
self.project.reference(lambda_version), function.name
],
lambda_arn=lambda_ref,
FunctionName=troposphere.Ref(function),
S3ObjectVersion=troposphere.Ref(
utils.valid_cloudformation_name(self.name, "s3version")),
))
alias = template.add_resource(
awslambda.Alias(
self.current_alias_cf_name,
DependsOn=[version.name],
FunctionName=troposphere.Ref(function),
FunctionVersion=troposphere.GetAtt(version, "Version"),
Name="current",
))
if self._get_true_false('cli-output', 't'):
template.add_output([
troposphere.Output(
utils.valid_cloudformation_name("Clioutput",
self.in_project_name),
Value=troposphere.Ref(alias),
)
])
email_target = os.environ.get("EMAIL_TARGET", "*****@*****.**")
recaptcha_secret = os.environ.get("RECAPTCHA_SECRET")
cdomain = parsed_domain.netloc.replace(
".", "-")[:24] # AWS names have a character limit
### SNS
send_to = sns.Subscription(Protocol="email", Endpoint=email_target)
topic = sns.Topic("EmailContactForm", Subscription=[send_to])
### Lambda
environment = {"ORIGIN_DOMAIN": domain, "TOPIC_ARN": Ref(topic)}
if recaptcha_secret:
environment["RECAPTCHA_SECRET"] = recaptcha_secret
lambda_environment = awslambda.Environment(Variables=environment)
sns_publish_policy = iam.Policy(
PolicyName=f"lambda-sns-publish-policy-contact-form-{cdomain}",
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "sns:Publish",
"Resource": "arn:aws:sns:*:*:*"
}]
},
)
lambda_logging_policy = iam.Policy(
arcgis_lambda = template.add_resource(
awslambda.Function(
lambda_name,
Code=awslambda.Code(S3Bucket=bucket, S3Key=key),
Description="Lambda task that updates {}".format(lambda_name),
FunctionName=lambda_name,
Handler="lambda.agol_update",
Runtime="python3.7",
Timeout=timeout,
Role=GetAtt(arcgis_role.title, "Arn"),
MemorySize=mem,
Environment=awslambda.Environment(
Variables={
"data_url": data_url,
"portal_url": portal_url,
"portal_user": portal_user,
"portal_password": portal_password,
"portal_item": id,
"file": file,
"format": frmt
})))
# create an EventBridge rule to kick off the lambda
arcgis_rule = template.add_resource(
events.Rule(rule_name,
ScheduleExpression=cron,
Description="My Lambda CloudWatch Event",
State="ENABLED",
Targets=[
events.Target("MyLambdaTarget",
Arn=GetAtt(arcgis_lambda.title, "Arn"),
Id="MyLambdaId")
awslambda.Function(
'LambdaBackupRDSFunction',
Description='Copies RDS backups to another region',
Code=awslambda.Code(
S3Bucket=Ref(s3_bucket_parameter),
S3Key=Ref(source_zip_parameter),
),
Handler='backup-rds.lambda_handler',
MemorySize=128,
Role=GetAtt(backup_rds_role, 'Arn'),
Runtime='python3.6',
Timeout=30,
Environment=awslambda.Environment(
Variables={
'SOURCE_REGION': Ref(AWS_REGION),
'TARGET_REGION': Ref(target_region_parameter),
'KMS_KEY_ID': Ref(kms_key_parameter),
'CLUSTERS_TO_USE': Ref(clusters_to_use_parameter)
})))
# SNS topic for event subscriptions
rds_topic = template.add_resource(
sns.Topic('RDSBackupTopic',
Subscription=[
sns.Subscription(
Protocol="lambda",
Endpoint=GetAtt(backup_rds_function, 'Arn'),
)
]))
# Event subscription - RDS will notify SNS when backup is started and finished
def createSQSConsumer(name, timeout=5, snsTopic=None):
res = {}
# create queue
res['QueueName'] = name + 'Queue'
queue = t.add_resource(
sqs.Queue(res['QueueName'], VisibilityTimeout=timeout * 2))
queueArn = GetAtt(res['QueueName'], 'Arn')
# create subscription
if (snsTopic):
res['SubscriptionName'] = name + 'Subscription'
subscription = t.add_resource(
sns.SubscriptionResource(
res['SubscriptionName'],
TopicArn=snsTopic,
Endpoint=queueArn,
Protocol='sqs',
RawMessageDelivery='true',
))
t.add_resource(
sqs.QueuePolicy(name + 'AllowSNS2SQSPolicy',
Queues=[queue.Ref()],
PolicyDocument={
"Version":
"2008-10-17",
"Id":
"PublicationPolicy",
"Statement": [{
"Sid": "Allow-SNS-SendMessage",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": ["sqs:SendMessage"],
"Resource": queue.GetAtt("Arn"),
"Condition": {
"ArnEquals": {
"aws:SourceArn": snsTopic
}
}
}]
}))
# create consumer function
res['FunctionName'] = name + 'Consumer'
consumer = t.add_resource(
serverless.Function(
res['FunctionName'],
Runtime=nodeRuntime,
CodeUri=lambdaSrcPath,
Handler=lambdaHandlerPath + name + 'Consumer.handler',
Timeout=timeout,
Events={
'SQSTrigger': {
'Type': 'SQS',
'Properties': {
'Queue': queueArn,
'BatchSize': 10
}
}
},
Policies=[
'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole',
{
'Version':
'2012-10-17',
'Statement': [{
'Effect':
'Allow',
'Action': [
'sqs:ReceiveMessage',
'sqs:ChangeMessageVisibility', 'sqs:DeleteMessage'
],
'Resource':
GetAtt(res['QueueName'], 'Arn')
}],
}
],
VpcConfig=lambdaVpcConfig,
Environment=awslambda.Environment(None, Variables=lambdaEnvVars),
))
return res
graphQL.CodeUri = lambdaSrcPath
graphQL.Handler = lambdaHandlerPath + 'GraphQL.handler'
graphQL.Events = {
'API': {
'Type': 'Api',
'Properties': {
'Path': '/graphql',
'Method': 'post'
}
}
}
graphQL.Policies = [
'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole'
]
graphQL.VpcConfig = lambdaVpcConfig
graphQL.Environment = awslambda.Environment(None, Variables=lambdaEnvVars)
t.add_resource(graphQL)
def createSQSConsumer(name, timeout=5, snsTopic=None):
res = {}
# create queue
res['QueueName'] = name + 'Queue'
queue = t.add_resource(
sqs.Queue(res['QueueName'], VisibilityTimeout=timeout * 2))
queueArn = GetAtt(res['QueueName'], 'Arn')
# create subscription
if (snsTopic):
res['SubscriptionName'] = name + 'Subscription'
)
lambda_handler = awslambda.Function(
"LambdaHandler",
Description="test lambda with access to RDS, SNS, DynamoDB. Will be triggered by SNS",
Handler="lambda.handler",
MemorySize=128,
Code=awslambda.Code(
S3Bucket=Ref(source_code_s3_bucket),
S3Key=Ref(source_code_s3_bucket_key),
),
Role=GetAtt(lambda_handler_role, "Arn"),
Runtime="python2.7",
Timeout=3,
VpcConfig=awslambda.VPCConfig(
SubnetIds=Ref(subnet_ids),
SecurityGroupIds=[GetAtt(lambda_security_group, "GroupId")],
),
Environment=awslambda.Environment(
Variables={
"DB_HOST": GetAtt(rds_instance, "Endpoint.Address"),
"DB_PORT": GetAtt(rds_instance, "Endpoint.Port"),
"DB_NAME": Ref(rds_db_name),
"DB_USER": Ref(rds_master_username),
"DB_PASSWORD": Ref(rds_master_password),
"SNS_TOPIC_ARN": emails_sns_topic,
"DYNAMO_DB_TABLE_NAME": Ref(dynamodb_table),
}
),
)
def create_template(self):
"""Create template (main function called by Stacker)."""
template = self.template
variables = self.get_variables()
template.set_version('2010-09-09')
template.set_description('App - Build Pipeline')
# Resources
boundary_arn = Join('', [
'arn:', Partition, ':iam::', AccountId, ':policy/',
variables['RolePermissionsBoundaryName'].ref
])
# Repo image limit is 1000 by default; this lambda function will prune
# old images
image_param_path = Join(
'', ['/', variables['AppPrefix'].ref, '/current-hash'])
image_param_arn = Join('', [
'arn:', Partition, ':ssm:', Region, ':', AccountId, ':parameter',
image_param_path
])
ecr_repo_arn = Join('', [
'arn:', Partition, ':ecr:', Region, ':', AccountId, ':repository/',
variables['EcrRepoName'].ref
])
cleanuplambdarole = template.add_resource(
iam.Role('CleanupLambdaRole',
AssumeRolePolicyDocument=make_simple_assume_policy(
'lambda.amazonaws.com'),
ManagedPolicyArns=[
IAM_ARN_PREFIX + 'AWSLambdaBasicExecutionRole'
],
PermissionsBoundary=boundary_arn,
Policies=[
iam.Policy(
PolicyName=Join(
'',
[variables['AppPrefix'].ref, '-ecrcleanup']),
PolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(Action=[awacs.ssm.GetParameter],
Effect=Allow,
Resource=[image_param_arn]),
Statement(Action=[
awacs.ecr.DescribeImages,
awacs.ecr.BatchDeleteImage
],
Effect=Allow,
Resource=[ecr_repo_arn])
]))
]))
cleanupfunction = template.add_resource(
awslambda.Function(
'CleanupFunction',
Description='Cleanup stale ECR images',
Code=awslambda.Code(
ZipFile=variables['ECRCleanupLambdaFunction']),
Environment=awslambda.Environment(
Variables={
'ECR_REPO_NAME': variables['EcrRepoName'].ref,
'SSM_PARAM': image_param_path
}),
Handler='index.handler',
Role=cleanuplambdarole.get_att('Arn'),
Runtime='python3.6',
Timeout=120))
cleanuprule = template.add_resource(
events.Rule('CleanupRule',
Description='Regularly invoke CleanupFunction',
ScheduleExpression='rate(7 days)',
State='ENABLED',
Targets=[
events.Target(Arn=cleanupfunction.get_att('Arn'),
Id='CleanupFunction')
]))
template.add_resource(
awslambda.Permission(
'AllowCWLambdaInvocation',
FunctionName=cleanupfunction.ref(),
Action=awacs.awslambda.InvokeFunction.JSONrepr(),
Principal='events.amazonaws.com',
SourceArn=cleanuprule.get_att('Arn')))
appsource = template.add_resource(
codecommit.Repository(
'AppSource',
RepositoryName=Join('-',
[variables['AppPrefix'].ref, 'source'])))
for i in ['Name', 'Arn']:
template.add_output(
Output("AppRepo%s" % i,
Description="%s of app source repo" % i,
Value=appsource.get_att(i)))
bucket = template.add_resource(
s3.Bucket(
'Bucket',
AccessControl=s3.Private,
LifecycleConfiguration=s3.LifecycleConfiguration(Rules=[
s3.LifecycleRule(NoncurrentVersionExpirationInDays=90,
Status='Enabled')
]),
VersioningConfiguration=s3.VersioningConfiguration(
Status='Enabled')))
template.add_output(
Output('PipelineBucketName',
Description='Name of pipeline bucket',
Value=bucket.ref()))
# This list must be kept in sync between the CodeBuild project and its
# role
build_name = Join('', [variables['AppPrefix'].ref, '-build'])
build_role = template.add_resource(
iam.Role(
'BuildRole',
AssumeRolePolicyDocument=make_simple_assume_policy(
'codebuild.amazonaws.com'),
PermissionsBoundary=boundary_arn,
Policies=[
iam.Policy(
PolicyName=Join('', [build_name, '-policy']),
PolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(
Action=[awacs.s3.GetObject],
Effect=Allow,
Resource=[
Join('', [bucket.get_att('Arn'), '/*'])
]),
Statement(
Action=[awacs.ecr.GetAuthorizationToken],
Effect=Allow,
Resource=['*']),
Statement(Action=[
awacs.ecr.BatchCheckLayerAvailability,
awacs.ecr.BatchGetImage,
awacs.ecr.CompleteLayerUpload,
awacs.ecr.DescribeImages,
awacs.ecr.GetDownloadUrlForLayer,
awacs.ecr.InitiateLayerUpload,
awacs.ecr.PutImage,
awacs.ecr.UploadLayerPart
],
Effect=Allow,
Resource=[ecr_repo_arn]),
Statement(Action=[
awacs.ssm.GetParameter,
awacs.ssm.PutParameter
],
Effect=Allow,
Resource=[image_param_arn]),
Statement(Action=[
awacs.logs.CreateLogGroup,
awacs.logs.CreateLogStream,
awacs.logs.PutLogEvents
],
Effect=Allow,
Resource=[
Join('', [
'arn:', Partition, ':logs:',
Region, ':', AccountId,
':log-group:/aws/codebuild/',
build_name
] + x)
for x in [[':*'], [':*/*']]
])
]))
]))
buildproject = template.add_resource(
codebuild.Project(
'BuildProject',
Artifacts=codebuild.Artifacts(Type='CODEPIPELINE'),
Environment=codebuild.Environment(
ComputeType='BUILD_GENERAL1_SMALL',
EnvironmentVariables=[
codebuild.EnvironmentVariable(
Name='AWS_DEFAULT_REGION',
Type='PLAINTEXT',
Value=Region),
codebuild.EnvironmentVariable(Name='AWS_ACCOUNT_ID',
Type='PLAINTEXT',
Value=AccountId),
codebuild.EnvironmentVariable(
Name='IMAGE_REPO_NAME',
Type='PLAINTEXT',
Value=variables['EcrRepoName'].ref),
],
Image='aws/codebuild/docker:18.09.0',
Type='LINUX_CONTAINER'),
Name=build_name,
ServiceRole=build_role.get_att('Arn'),
Source=codebuild.Source(
Type='CODEPIPELINE',
BuildSpec=variables['BuildProjectBuildSpec'])))
pipelinerole = template.add_resource(
iam.Role(
'PipelineRole',
AssumeRolePolicyDocument=make_simple_assume_policy(
'codepipeline.amazonaws.com'),
PermissionsBoundary=boundary_arn,
Policies=[
iam.Policy(
PolicyName=Join('', [build_name, '-pipeline-policy']),
PolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(
Action=[
awacs.codecommit.GetBranch,
awacs.codecommit.GetCommit,
awacs.codecommit.UploadArchive,
awacs.codecommit.
GetUploadArchiveStatus, # noqa
awacs.codecommit.CancelUploadArchive
], # noqa
Effect=Allow,
Resource=[appsource.get_att('Arn')]),
Statement(
Action=[awacs.s3.GetBucketVersioning],
Effect=Allow,
Resource=[bucket.get_att('Arn')]),
Statement(
Action=[
awacs.s3.GetObject, awacs.s3.PutObject
],
Effect=Allow,
Resource=[
Join('', [bucket.get_att('Arn'), '/*'])
]),
Statement(
Action=[
awacs.codebuild.BatchGetBuilds,
awacs.codebuild.StartBuild
],
Effect=Allow,
Resource=[buildproject.get_att('Arn')])
]))
]))
template.add_resource(
codepipeline.Pipeline(
'Pipeline',
ArtifactStore=codepipeline.ArtifactStore(Location=bucket.ref(),
Type='S3'),
Name=build_name,
RoleArn=pipelinerole.get_att('Arn'),
Stages=[
codepipeline.Stages(
Name='Source',
Actions=[
codepipeline.Actions(
Name='CodeCommit',
ActionTypeId=codepipeline.ActionTypeId(
Category='Source',
Owner='AWS',
Provider='CodeCommit',
Version='1'),
Configuration={
'RepositoryName':
appsource.get_att('Name'), # noqa
'BranchName': 'master'
},
OutputArtifacts=[
codepipeline.OutputArtifacts(
Name='CodeCommitRepo')
]),
]),
codepipeline.Stages(
Name='Build',
Actions=[
codepipeline.Actions(
Name='Build',
ActionTypeId=codepipeline.ActionTypeId(
Category='Build',
Owner='AWS',
Provider='CodeBuild',
Version='1'),
Configuration={
'ProjectName': buildproject.ref()
},
InputArtifacts=[
codepipeline.InputArtifacts(
Name='CodeCommitRepo')
])
])
]))
# ============================================================================
# Lambda Function
# ============================================================================
lambda_processing = awslambda.Function(
'LambdaProcessing',
Handler='lambda_function.lambda_handler',
Role=GetAtt("LambdaExecutionRole", "Arn"),
Runtime='python3.7',
FunctionName=Ref(lambda_name),
Code=awslambda.Code(S3Bucket=Ref(source_code_bucket),
S3Key=Ref(zip_file_path)),
Environment=awslambda.Environment(
Variables={
'DB_HOST': GetAtt(db, 'Endpoint.Address'),
'DB_NAME': Ref(db_name),
'DB_PASSWORD': Ref(db_password),
'DB_PORT': GetAtt(db, 'Endpoint.Port'),
'DB_USERNAME': Ref(db_user),
'DYNAMODB_TABLE': Ref(dynamo_db),
'SNS_TOPIC': Ref(sns_topic),
}))
# ============================================================================
# Lambda Permission
# ============================================================================
s3_lambda_permission = awslambda.Permission(
'S3LambdaPermission',
Action='lambda:invokeFunction',
Principal='s3.amazonaws.com',
FunctionName=Ref(lambda_processing),
)
Resource=['*'])
]))
]))
update_security_groups_function = t.add_resource(
awslambda.Function(
'UpdateSecurityGroups',
Description='Update Beanstalk Environment SecurityGroup',
# code=awslambda.code(
# S3Bucket=Sub('aws-lambda-code-${AWS::AccountId}-${AWS::Region}'),
# S3Key='update_security_groups.py.zip'
# ),
Code=awslambda.Code(ZipFile=cfnutil.load_python_lambda(
'lambdas/update_security_groups.py')),
Environment=awslambda.Environment(
Variables={'REGIONS': Ref(param_regions)}),
Handler='index.lambda_handler',
Role=GetAtt(update_function_execution_role, 'Arn'),
Runtime='python2.7',
MemorySize='128',
Timeout='300',
))
ip_space_changed_topic = 'arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged'
update_function_lambda_permission = t.add_resource(
awslambda.Permission('LambdaPermission',
FunctionName=Ref(update_security_groups_function),
Action='lambda:InvokeFunction',
Principal='sns.amazonaws.com',
SourceArn=ip_space_changed_topic))
'HelloWorldEncVarsFunction',
FunctionName='hello_world_enc_vars',
Description='Hello world lambdas with encrypted vars Python 3.6',
Handler='lambda_function.lambda_handler',
Role=GetAtt('ExecutionRole', 'Arn'),
KmsKeyArn=Ref(kms_arn),
Code=awslambda.Code(
S3Bucket='nicor-dev',
S3Key='deployments/lambdas/travis_build/hello_world.zip',
),
Runtime='python3.6',
Timeout='30',
MemorySize=128,
Environment=awslambda.Environment(
'LambdaVars',
Variables={
'SAMPLE_VAR': encrypt(kms_alias='nicor88-key', value='test')
#replace with something from secret env
})))
# utils.add_lambda_scheduler(template_res=template,
# lambda_function_name='hello_world',
# lambda_function_arn=GetAtt(hello_world_lambda, 'Arn'),
# cron='cron(0/5 * * * ? *)'
# )
template.add_output([
Output('LambdaExecutionRole',
Description='Lambdas Execution role',
Value=Ref(lambda_execution_role)),
Output('HelloWorldLambda',
Description='HelloWorld Lambda Function',
'KinesisStreamToFirehose',
FunctionName=cfg['lambda_function_name'],
Description=
'Lambda function to read kinesis stream and put to firehose',
Handler='lambda_function.lambda_handler',
Role=GetAtt('ExecutionRole', 'Arn'),
Code=awslambda.Code(
S3Bucket=cfg['s3_deployment_bucket'],
S3Key=cfg['s3_key_lambda'],
),
Runtime='python3.6',
Timeout=cfg['lambda_timeout'],
MemorySize=cfg['lambda_memory_size'],
Environment=awslambda.Environment(
'LambdaVars',
Variables={
'DELIVERY_STREAM': cfg['kinesis_delivery_stream_name'],
'ADD_NEWLINE': 'True'
})))
add_kinesis_trigger_for_lambda = template.add_resource(
awslambda.EventSourceMapping(
'KinesisLambdaTrigger',
BatchSize=cfg['lambda_batch_size'],
Enabled=cfg['lambda_enabled'],
FunctionName=Ref(lambda_stream_to_firehose),
StartingPosition=cfg['lambda_starting_position'],
EventSourceArn=GetAtt(kinesis_stream, 'Arn')))
template_json = template.to_json(indent=4)
print(template_json)
def _lambda_environment(bucket_name: Parameter) -> awslambda.Environment:
return awslambda.Environment(Variables=dict(S3_BUCKET=bucket_name.ref()))
def environment(self):
environment = self.get_variables()["Environment"]
env = NoValue
if environment:
env = awslambda.Environment(Variables=environment)
return env
def lambda_configuration(name):
resources = []
"""
define variables
"""
select_function = name
print("passing function name to configuration: {}".format(name))
bucket_name = lambda_config['GLOBAL']['BUCKETNAME']
dynamodb_table = lambda_config['DynamoDB']['table_name']
source_file_name = lambda_config['Lambda']['functions'][select_function]['source_file_name']
title = lambda_config['Lambda']['functions'][select_function]['title']
load_packages = lambda_config['Lambda']['functions'][select_function]['load_packages']
assigned_roles = [iam.allow_lambda_execute, # REQUIRED TO ENABLE LOGGING
iam.allow_dynamodb_action]
function_name = lambda_config['Lambda']['functions'][select_function]['function_name']
function_description = lambda_config['Lambda']['functions'][select_function]['function_description']
function_to_run = lambda_config['Lambda']['functions'][select_function]['function_to_run']
memory_size = int(lambda_config['Lambda']['functions'][select_function]['memory_size'])
timeout = lambda_config['Lambda']['functions'][select_function]['timeout']
environment_var = lmb.Environment(
Variables={"SITE": "https://aws.amazon.com/console/",
"EXPECTED":"AWS"}
)
if memory_size > 1536:
sys.exit("Memory Size too large")
if memory_size < 128:
sys.exit("Memory Size too small")
if memory_size % 64 != 0:
sys.exit("Memory Size not a multiple of 64")
source = 'lambda/{0}'.format(source_file_name)
file_name_only = os.path.splitext(os.path.basename(source))[0]
zip_file = file_name_only + '.zip'
function_handler = file_name_only + '.' + function_to_run
role_title = '{0}Role'.format(title)
code_title = '{0}Code'.format(title)
function_title = '{0}Function'.format(title)
permission_title = '{0}Permission'.format(title)
target_id = '{0}Target'.format(title)
# Copy the source file into the S3 Bucket
s3.zip_and_load(source, load_packages=load_packages)
role = Role(
role_title,
AssumeRolePolicyDocument=iam.lambda_assume_role,
Policies=assigned_roles
)
resources.append(role)
code = lmb.Code(
code_title,
S3Bucket=bucket_name,
S3Key=zip_file
)
function = lmb.Function(
function_title,
FunctionName=function_name,
Description=function_description,
Code=code,
Handler=function_handler,
MemorySize=memory_size,
Role=GetAtt(role_title, "Arn"),
Environment=environment_var,
Runtime='python3.6',
Timeout=timeout
)
resources.append(function)
permission = lmb.Permission(
permission_title,
FunctionName=function_name,
Action="lambda:InvokeFunction",
Principal="events.amazonaws.com",
DependsOn=function_title
)
resources.append(permission)
print("type resources is: {} ".format(type(resources)))
target = Target(
Arn=GetAtt(function, "Arn"),
Id=target_id
)
print("type target is: {}".format(target))
"""we need to return both: list of resources and target. The last will be used for CW rule """
return [resources, target]
def add_lambda_function(self):
role = self.add_resource(
iam.Role(
'TerraformRegistryLambdaRole',
AssumeRolePolicyDocument=PolicyDocument(
Version='2012-10-17',
Statement=[
Statement(Effect=Allow,
Action=[Action('sts', 'AssumeRole')],
Principal=Principal('Service',
'lambda.amazonaws.com'))
]),
ManagedPolicyArns=[
'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
],
Policies=[
iam.Policy(
PolicyName='Registry',
PolicyDocument=PolicyDocument(Statement=[
Statement(Effect=Allow,
Action=[Action('s3', '*')],
Resource=[
GetAtt(self._bucket, 'Arn'),
Join('', [
GetAtt(self._bucket, 'Arn'), '/*'
])
]),
Statement(Effect=Allow,
Action=[Action('dynamodb', '*')],
Resource=[
GetAtt(self._api_token_table, 'Arn')
])
]))
]))
lambda_function = self.add_resource(
awslambda.Function(
'TerraformRegistry',
Runtime='python3.7',
Code=awslambda.Code(S3Bucket=LAMBDA_PACKAGE_BUCKET,
S3Key=f'{self._build_version}/lambda.zip'),
Handler='registry.handler',
Timeout=300,
Role=GetAtt(role, 'Arn'),
Description=Sub('${AWS::StackName} Terraform Registry'),
Environment=awslambda.Environment(
Variables={
'TerraformModules': Ref(self._bucket),
'ApiTokens': Ref(self._api_token_table)
})))
aws_sha256, hex_sha256 = sha256('build/lambda.zip')
version_name = 'TerraformRegistryVersion' + hex_sha256
self._lambda_function = self.add_resource(
awslambda.Version(version_name,
CodeSha256=aws_sha256,
Description=hex_sha256,
FunctionName=Ref(lambda_function),
DependsOn=[lambda_function],
DeletionPolicy=Retain))
lambdafunc.Handler = "main.main"
lambdafunc.MemorySize = 128
lambdafunc.Role = GetAtt(lambdaexecutionrole.title, "Arn")
lambdafunc.Runtime = "python3.6"
lambdafunc.Environment = awslambda.Environment(Variables={
"JICKET_IMAP_HOST": jicketimaphost,
"JICKET_IMAP_PORT": jicketimapport,
"JICKET_IMAP_USER": jicketimapuser,
"JICKET_IMAP_PASS": jicketimappass,
"JICKET_SMTP_HOST": jicketsmtphost,
"JICKET_SMTP_PORT": jicketsmtpport,
"JICKET_SMTP_USER": jicketsmtpuser,
"JICKET_SMTP_PASS": jicketsmtppass,
"JICKET_JIRA_URL": jicketjiraurl,
"JICKET_JIRA_PROJECT": jicketjiraproject,
"JICKET_JIRA_USER": jicketjirauser,
"JICKET_JIRA_PASS": jicketjirapass,
"JICKET_THREAD_TEMPLATE": jicketthreadtemplate,
"JICKET_TICKET_ADDRESS": jicketticketaddress,
"JICKET_ID_PREFIX": jicketidprefix,
"JICKET_LOOPMODE": jicketloopmode,
})
t.add_resource(lambdafunc)
