def create_sns(alarm_email):
email_topic = Topic('email_notification',
Subscription=[
Subscription(Endpoint=Ref(alarm_email),
Protocol="email")
])
return email_topic
def create_sns_topic(stack, name, endpoint, protocol='https'):
"""Add a SNS topic."""
return stack.stack.add_resource(
Topic(
'{0}Topic'.format(name.replace('-', '')),
DisplayName=name,
Subscription=[Subscription(Endpoint=endpoint, Protocol=protocol)],
TopicName='{0}Topic'.format(name)))
def topic(topic_title, emails):
topic = Topic(topic_title,
DisplayName=Join("",
[Ref("AWS::StackName"), "-", topic_title]))
topic.Subscription = []
for index, email in enumerate(emails):
topic.Subscription.append(
Subscription(topic_title + "Subscription" + str(index),
Endpoint=email,
Protocol="email"))
return topic
def set_sqs_topic(subscription, content):
"""
Function to set permissions and import for SQS subscription
:return:
"""
if keypresent(ENDPOINT_KEY, subscription) and not subscription[
ENDPOINT_KEY
].startswith("arn:"):
assert check_queue_exists(subscription[ENDPOINT_KEY], content)
endpoint = (
get_import_value(subscription[ENDPOINT_KEY], SQS_ARN_T)
if not subscription[ENDPOINT_KEY].startswith("arn:")
else subscription[ENDPOINT_KEY]
)
return Subscription(Protocol="sqs", Endpoint=endpoint)
def add_subscription(self, endpoint, protocol):
"""
Adds a subscription to this sns topic.
AWS: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sns-subscription.html
Troposphere: https://github.com/cloudtools/troposphere/blob/master/troposphere/sns.py
:param endpoint: the endpoint to send the notification to (eg '*****@*****.**')
:param protocol: the protocol to use to send the notification (eg 'email')
"""
sub = Subscription('{0}Subscription{1}'.format(
self.trop_topic.title, str(len(self.subscriptions))),
Endpoint=endpoint,
Protocol=protocol)
self.subscriptions.append(sub)
self.trop_topic.Subscription = self.subscriptions
def create_sns_sqs(template, sns_name, sqs_name):
q = template.add_resource(Queue(sqs_name, QueueName=sqs_name))
topic = template.add_resource(
Topic(sns_name,
TopicName=sns_name,
Subscription=[
Subscription(Endpoint=GetAtt(q, 'Arn'), Protocol='sqs')
]))
policy = template.add_resource(
QueuePolicy(sqs_name + sns_name + 'policy',
PolicyDocument={
"Version":
"2012-10-17",
"Id":
"MyQueuePolicy",
"Statement": [{
"Sid": "Allow-SendMessage-From-SNS-Topic",
"Effect": "Allow",
"Principal": "*",
"Action": ["sqs:SendMessage"],
"Resource": "*",
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(topic)
}
}
}]
},
Queues=[sqs_name]))
template.add_output(Output("sns", Description="SNS Arn", Value=Ref(topic)))
template.add_output(
Output("queuearn", Description="Queue Arn", Value=GetAtt(q, 'Arn')))
template.add_output(
Output("queueurl", Description="Queue URL", Value=Ref(q)))
raise Exception("Clean ES function too long! Has " + str(len(content)))
clea_es_function = t.add_resource(
Function('LambdaCleanESFunction',
Description='Removes old ElasticSearch indexes',
Code=Code(ZipFile=content),
Handler='index.lambda_handler',
MemorySize=128,
Role=GetAtt(es_exec_role, 'Arn'),
Runtime='python2.7',
Timeout=60))
alarm_topic = t.add_resource(
Topic('LambdaErrorTopic',
Subscription=[
Subscription(Protocol="email", Endpoint=Ref(param_alarm_email))
]))
t.add_resource(
Alarm("LambdaBaseErrorsAlarm",
ComparisonOperator='GreaterThanThreshold',
EvaluationPeriods=1,
MetricName='Errors',
Namespace='AWS/Lambda',
Dimensions=[
MetricDimension(Name='FunctionName', Value=Ref(base_function))
],
Period=300,
Statistic='Maximum',
Threshold='0',
AlarmActions=[Ref(alarm_topic)]))
from template import t
from troposphere import GetAtt, Ref, Sub
from troposphere.awslambda import Permission
from troposphere.sns import Subscription, Topic
from troposphere.ssm import Parameter
from .hyp3_send_email import send_email
print(' adding sns')
finish_sns = t.add_resource(Topic(
"HyP3FinishEventSNSTopic",
Subscription=[
Subscription(
Protocol="lambda",
Endpoint=GetAtt(send_email, "Arn")
)
]
))
sns_invoke_permissions = t.add_resource(Permission(
"SNSSchedulerInvokePermissions",
Action="lambda:InvokeFunction",
Principal="sns.amazonaws.com",
SourceArn=Ref(finish_sns),
FunctionName=GetAtt(send_email, "Arn")
))
ssm_sns_arn = t.add_resource(Parameter(
"HyP3SSMParameterFinishEventSNSArn",
Name=Sub(
))
template.add_resource(
LogGroup(
"RequestEncodingLambdaLogGroup",
LogGroupName=Join(
'/', ['/aws/lambda', Ref(request_encoding_function)]),
RetentionInDays=7,
))
request_encoding_topic = template.add_resource(
Topic(
'RequestEncodingTopic',
Subscription=[
Subscription(
Protocol='sqs',
Endpoint=GetAtt(request_encoding_queue, 'Arn'),
),
Subscription(
Protocol='lambda',
Endpoint=GetAtt(request_encoding_function, 'Arn'),
),
Subscription(
Protocol='sqs',
Endpoint=GetAtt(start_media_insights_queue, 'Arn'),
)
],
))
template.add_resource(
Permission(
'InvokeRequestEncodingFunctionPermission',
from troposphere import GetAtt, Output, Ref, Template
from troposphere.sns import Subscription, Topic
from troposphere.sqs import Queue, QueuePolicy
t = Template()
t.set_description("AWS CloudFormation Sample Template: This template "
"demonstrates the creation of a DynamoDB table.")
sqsqueue = t.add_resource(Queue("SQSQueue"))
snstopic = t.add_resource(
Topic("SNSTopic",
Subscription=[
Subscription(Protocol="sqs", Endpoint=GetAtt(sqsqueue, "Arn"))
]))
t.add_output(
Output(
"QueueArn",
Value=GetAtt(sqsqueue, "Arn"),
Description="ARN of SQS Queue",
))
t.add_resource(
QueuePolicy("AllowSNS2SQSPolicy",
Queues=[Ref(sqsqueue)],
PolicyDocument={
"Version":
"2008-10-17",
))
template.add_parameter_to_group(start_insights_code_key, 'Lambda Keys')
template.add_parameter_to_group(rekognition_code_key, 'Lambda Keys')
template.add_parameter_to_group(video_metadata_event_code_key, 'Lambda Keys')
template.add_parameter_to_group(rekognition_results_code_key, 'Lambda Keys')
rekognition_updates_queue = template.add_resource(
Queue('RekognitionUpdatesQueue', ))
rekognition_updates_topic = template.add_resource(
Topic(
'RekognitionUpdatesTopic',
Subscription=[
Subscription(
Endpoint=GetAtt(rekognition_updates_queue, 'Arn'),
Protocol='sqs',
)
],
))
template.add_resource(
QueuePolicy(
"RekognitionUpdatesQueuePolicy",
Queues=[Ref(rekognition_updates_queue)],
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["sqs:SendMessage"],
"Resource": GetAtt(rekognition_updates_queue, 'Arn'),
def emit_configuration():
# Build an SQS queue for the babysitter
"""create_queue = template.add_parameter(
Parameter(
'CreateDeregistrationTopic',
Type='String',
Description='Whether or not to create the Chef Deregistration queue. This option is provided in case the queue already exists.',
Default='no',
AllowedValues=['yes', 'no'],
ConstraintDescription='Answer must be yes or no'
)
)
conditions = {
"CreateDeregCondition": Equals(
Ref(create_queue), "yes"
)
}
for c in conditions:
template.add_condition(c, conditions[c])"""
queue_name = '_'.join(['chef-deregistration', CLOUDNAME, CLOUDENV])
queue = template.add_resource(
Queue(
cfn.sanitize_id(queue_name),
VisibilityTimeout=60,
MessageRetentionPeriod=1209600,
MaximumMessageSize=16384,
QueueName=queue_name,
))
alert_topic = template.add_resource(
Topic(
cfn.sanitize_id("BabysitterAlarmTopic{0}".format(CLOUDENV)),
DisplayName='Babysitter Alarm',
TopicName=queue_name,
Subscription=[
Subscription(Endpoint=GetAtt(queue, "Arn"), Protocol='sqs'),
],
DependsOn=queue.title,
))
queue_depth_alarm = template.add_resource(
Alarm(
"BabysitterQueueDepthAlarm",
AlarmDescription=
'Alarm if the queue depth grows beyond 200 messages',
Namespace='AWS/SQS',
MetricName='ApproximateNumberOfMessagesVisible',
Dimensions=[
MetricDimension(Name='QueueName',
Value=GetAtt(queue, "QueueName"))
],
Statistic='Sum',
Period='300',
EvaluationPeriods='1',
Threshold='200',
ComparisonOperator='GreaterThanThreshold',
#AlarmActions=[Ref(alert_topic), ],
#InsufficientDataActions=[Ref(alert_topic), ],
DependsOn=alert_topic.title,
), )
queue_policy = {
"Version":
"2012-10-17",
"Id":
"BabysitterSNSPublicationPolicy",
"Statement": [{
"Sid": "AllowSNSPublishing",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": ["sqs:SendMessage"],
"Resource": GetAtt(queue, "Arn"),
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(alert_topic)
}
}
}]
}
# Publish all events from SNS to the Queue
template.add_resource(
QueuePolicy(
"BabysitterPublishSNStoSQSPolicy",
Queues=[Ref(queue)],
PolicyDocument=queue_policy,
DependsOn=[queue.title, alert_topic.title],
))
cfn.alert_topic = alert_topic
OperatorEmail = t.add_parameter(Parameter(
"OperatorEmail",
Type="String",
Description="Email address to notify when new logs are published.",
))
S3Bucket = t.add_resource(Bucket(
"S3Bucket",
DeletionPolicy="Retain"
))
Topic = t.add_resource(Topic(
"Topic",
Subscription=[
Subscription(
Endpoint=Ref(OperatorEmail),
Protocol="email",
),
],
))
TopicPolicy = t.add_resource(TopicPolicy(
"TopicPolicy",
Topics=[Ref(Topic)],
PolicyDocument={
"Version": "2008-10-17", "Statement": [{
"Action": "SNS:Publish",
"Principal": {
"Service": "cloudtrail.amazonaws.com"
},
"Resource": "*", "Effect": "Allow", "Sid": "AWSCloudTrailSNSPolicy"
}]
from troposphere.ec2 import PortRange
from troposphere.sns import Topic, Subscription
t = Template()
t.add_version("2010-09-09")
t.add_description("""\
SNS""")
Primary = t.add_resource(
Topic(
"Primary",
DisplayName="ShelterMutual-Primary",
TopicName="ShelterMutual-Primary",
Subscription=[
Subscription(
Endpoint="*****@*****.**",
Protocol="email",
),
],
))
PrimaryArn = t.add_output(
Output(
"PrimaryArn",
Description="Primary SNS ARN.",
Value=Ref(Primary),
))
print(t.to_json())
Name="pVideoDashboardTrigger" + suffixcf,
ScheduleExpression="cron(0/5 8-17 ? * MON-FRI *)",
# Run every 5 minutes Monday through Friday between 8:00 am and 19:55 pm (UTC)
Description=
"cron for trigger extraction metrics by lambda (pVideoDashboardMetricsExtract)",
State=lambda_trigger_status,
Targets=[event_target]))
#### CLOUDWATCH ALARM ####
alarm_topic = t.add_resource(
Topic(
"pVideoDashboardAlarmTopic" + suffixcf,
TopicName="pVideoDashboardAlarmTopic" + suffixcf,
Subscription=[
Subscription(Endpoint=alarm_email_1, Protocol="email"),
Subscription(Endpoint=alarm_email_2, Protocol="email"),
],
))
queue_depthalarm = t.add_resource(
Alarm(
"pVideoDashboardAlarm" + suffixcf,
AlarmName="pVideoDashboardAlarm" + suffixcf,
AlarmDescription="pVideoDashboardAlarm if Error more than 2",
Namespace=
"AWS/Lambda", # The namespace of the metric that is associated with the alarm.
MetricName="Errors",
Dimensions=[ # The dimensions of the metric for the alarm.
MetricDimension(
Name="FunctionName",
dead_letter_queue = template.add_resource(
Queue("deadSQS", QueueName="dead__letter_queue__iac_sqs_sample"))
sqs_aws = template.add_resource(
Queue("iacSQS",
QueueName="iac_sqs_sample",
RedrivePolicy=RedrivePolicy(
deadLetterTargetArn=GetAtt(dead_letter_queue, "Arn"),
maxReceiveCount="5",
)))
sns_aws = template.add_resource(
Topic("iacSNS",
TopicName="iac_sns_sample",
Subscription=[
Subscription(Protocol="sqs", Endpoint=GetAtt(sqs_aws, "Arn"))
]))
template.add_output([
Output("SourceQueueURL",
Description="URL of the source queue",
Value=Ref(sqs_aws)),
Output("SourceQueueARN",
Description="ARN of the source queue",
Value=GetAtt(sqs_aws, "Arn")),
Output("DeadLetterQueueURL",
Description="URL of the dead letter queue",
Value=Ref(dead_letter_queue)),
Output("DeadLetterQueueARN",
Description="ARN of the dead letter queue",
Value=GetAtt(dead_letter_queue, "Arn")),
def generate(env='pilot'):
template = Template()
template.set_version("2010-09-09")
# ExistingVPC = template.add_parameter(Parameter(
# "ExistingVPC",
# Type="AWS::EC2::VPC::Id",
# Description=(
# "The VPC ID that includes the security groups in the"
# "ExistingSecurityGroups parameter."
# ),
# ))
#
# ExistingSecurityGroups = template.add_parameter(Parameter(
# "ExistingSecurityGroups",
# Type="List<AWS::EC2::SecurityGroup::Id>",
# ))
param_spider_lambda_memory_size = template.add_parameter(
Parameter(
'SpiderLambdaMemorySize',
Type=NUMBER,
Description='Amount of memory to allocate to the Lambda Function',
Default='128',
AllowedValues=MEMORY_VALUES
)
)
param_spider_lambda_timeout = template.add_parameter(
Parameter(
'SpiderLambdaTimeout',
Type=NUMBER,
Description='Timeout in seconds for the Lambda function',
Default='60'
)
)
spider_tasks_queue_dlq_name = f'{env}-spider-tasks-dlq'
spider_tasks_queue_dlq = template.add_resource(
Queue(
"SpiderTasksDLQ",
QueueName=spider_tasks_queue_dlq_name,
MessageRetentionPeriod=(60 * 60 * 24 * 14),
)
)
spider_tasks_queue_name = f"{env}-spider-tasks"
spider_tasks_queue = template.add_resource(
Queue(
"SpiderTasksQueue",
QueueName=spider_tasks_queue_name,
MessageRetentionPeriod=(60 * 60 * 24 * 14),
VisibilityTimeout=300,
RedrivePolicy=RedrivePolicy(
deadLetterTargetArn=GetAtt(spider_tasks_queue_dlq, "Arn"),
maxReceiveCount=2,
),
DependsOn=[spider_tasks_queue_dlq],
)
)
spider_lambda_role = template.add_resource(
Role(
"SpiderLambdaRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="root",
Statement=[
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("logs", "*")
]
),
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("s3", "*")
]
),
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("sqs", "*")
]
),
]
),
)
],
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
}
}]
},
)
)
spider_file_path = './spider/index.js'
spider_code = open(spider_file_path, 'r').readlines()
spider_lambda = template.add_resource(
Function(
"SpiderLambda",
Code=Code(
S3Bucket='spider-lambda',
S3Key=f'{env}.zip',
# ZipFile=Join("", spider_code)
),
Handler="index.handler",
Role=GetAtt(spider_lambda_role, "Arn"),
Runtime="nodejs12.x",
Layers=['arn:aws:lambda:us-east-1:342904801388:layer:spider-node-browser:1'],
MemorySize=Ref(param_spider_lambda_memory_size),
Timeout=Ref(param_spider_lambda_timeout),
DependsOn=[spider_tasks_queue],
)
)
# AllSecurityGroups = template.add_resource(CustomResource(
# "AllSecurityGroups",
# List=Ref(ExistingSecurityGroups),
# AppendedItem=Ref("SecurityGroup"),
# ServiceToken=GetAtt(spider_lambda, "Arn"),
# ))
#
# SecurityGroup = template.add_resource(SecurityGroup(
# "SecurityGroup",
# SecurityGroupIngress=[
# {"ToPort": "80", "IpProtocol": "tcp", "CidrIp": "0.0.0.0/0",
# "FromPort": "80"}],
# VpcId=Ref(ExistingVPC),
# GroupDescription="Allow HTTP traffic to the host",
# SecurityGroupEgress=[
# {"ToPort": "80", "IpProtocol": "tcp", "CidrIp": "0.0.0.0/0",
# "FromPort": "80"}],
# ))
#
# AllSecurityGroups = template.add_output(Output(
# "AllSecurityGroups",
# Description="Security Groups that are associated with the EC2 instance",
# Value=Join(", ", GetAtt(AllSecurityGroups, "Value")),
# ))
source_sns_name = f'{env}-source-sns-topic'
source_sns_topic = template.add_resource(
Topic(
"SNSSource",
TopicName=source_sns_name,
Subscription=[
Subscription(
Endpoint=GetAtt(spider_tasks_queue, "Arn"),
Protocol='sqs',
)
],
DependsOn=[spider_tasks_queue]
)
)
source_sns_topic_policy = template.add_resource(
TopicPolicy(
"SourceForwardingTopicPolicy",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="AllowS3PutMessageInSNS",
Statement=[
Statement(
Sid="AllowS3PutMessages",
Principal=Principal("Service", "s3.amazonaws.com"),
Effect=Allow,
Action=[
Action("sns", "Publish"),
],
Resource=["*"],
)
]
),
Topics=[Ref(source_sns_topic)],
)
)
sns_sqs_policy = template.add_resource(
QueuePolicy(
"AllowSNSPutMessagesInSQS",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="AllowSNSPutMessagesInSQS",
Statement=[
Statement(
Sid="AllowSNSPutMessagesInSQS2",
Principal=Principal("*"),
Effect=Allow,
Action=[
Action("sqs", "SendMessage"),
],
Resource=["*"],
)
]
),
Queues=[Ref(spider_tasks_queue)],
DependsOn=[spider_tasks_queue],
)
)
# Buckets
source_bucket_name = f'{env}-source-bucket'
source_bucket = template.add_resource(
Bucket(
"SourceBucket",
BucketName=source_bucket_name,
NotificationConfiguration=NotificationConfiguration(
TopicConfigurations=[
TopicConfigurations(
Topic=Ref(source_sns_topic),
Event="s3:ObjectCreated:*",
)
],
),
DependsOn=[source_sns_topic_policy],
)
)
results_bucket_name = f'{env}-results-bucket'
results_bucket = template.add_resource(
Bucket(
"ResultsBucket",
BucketName=results_bucket_name,
)
)
# Lambda trigger
template.add_resource(
EventSourceMapping(
"TriggerLambdaSpiderFromSQS",
EventSourceArn=GetAtt(spider_tasks_queue, "Arn"),
FunctionName=Ref(spider_lambda),
BatchSize=1, # Default process tasks one by one
)
)
return template.to_json()
apiKey = t.add_resource(api.ApiKey(
"apiKey",
StageKeys=[api.StageKey(
RestApiId=Ref(restApi),
StageName=Ref(apiStage)
)]
))
snsTopic = t.add_resource(Topic(
"snsTopic",
DisplayName="GDACK",
TopicName="GDACK",
Subscription=[
Subscription(
Endpoint=GetAtt(lambdaFunction, "Arn"),
Protocol="lambda"
)
]
))
ApiPermission = t.add_resource(Permission(
"ApiPermission",
FunctionName=Ref(lambdaFunction),
Action="lambda:InvokeFunction",
Principal="apigateway.amazonaws.com",
SourceArn=Join("", ["arn:aws:execute-api:us-west-2:", Ref("AWS::AccountId"), ":", Ref(restApi), "/*/*/*"])
))
# Output
WebhookUrl = t.add_output(Output(
"operational issues",
Type="String",
))
lambda_function_name = t.add_parameter(
Parameter(
"LambdaFunctionName",
Description="The name of the lambda function",
Type="String",
))
alarm_topic = t.add_resource(
Topic(
"ChaosLambdaAlarmTopic",
Subscription=[
Subscription(Endpoint=Ref(alarm_email), Protocol="email"),
],
))
t.add_resource(
Alarm(
"ChaosLambdaErrorAlarm",
AlarmName="chaosLambda/LambdaError",
AlarmDescription="Enters ALARM state because we have received a lamdba "
"error. See 'Errors' section on the following link: "
"http://docs.aws.amazon.com/lambda/latest/dg/"
"monitoring-functions-metrics.html for more "
"information.",
Namespace="AWS/Lambda",
MetricName="Errors",
Dimensions=[