def _queue_policy(self, template: Template, queue, queue_name: str,
subscriptions: dict):
template.add_resource(
QueuePolicy(f'{queue_name}Policy',
Queues=[Ref(queue)],
PolicyDocument={
'Version':
'2008-10-17',
'Id':
f'{queue_name}Policy',
'Statement': [{
'Action': [
'sqs:SendMessage',
],
'Effect': 'Allow',
'Resource': GetAtt(queue, 'Arn'),
'Principal': {
'AWS': '*',
},
'Condition': {
'ForAnyValue:ArnEquals': {
'aws:SourceArn': [
self._topic_arn(name)
for name in subscriptions.keys()
]
}
}
}]
},
DependsOn=queue))
def create_sns_sqs(template, sns_name, sqs_name):
q = template.add_resource(Queue(sqs_name, QueueName=sqs_name))
topic = template.add_resource(
Topic(sns_name,
TopicName=sns_name,
Subscription=[
Subscription(Endpoint=GetAtt(q, 'Arn'), Protocol='sqs')
]))
policy = template.add_resource(
QueuePolicy(sqs_name + sns_name + 'policy',
PolicyDocument={
"Version":
"2012-10-17",
"Id":
"MyQueuePolicy",
"Statement": [{
"Sid": "Allow-SendMessage-From-SNS-Topic",
"Effect": "Allow",
"Principal": "*",
"Action": ["sqs:SendMessage"],
"Resource": "*",
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(topic)
}
}
}]
},
Queues=[sqs_name]))
template.add_output(Output("sns", Description="SNS Arn", Value=Ref(topic)))
template.add_output(
Output("queuearn", Description="Queue Arn", Value=GetAtt(q, 'Arn')))
template.add_output(
Output("queueurl", Description="Queue URL", Value=Ref(q)))
)
],
))
template.add_resource(
QueuePolicy(
"StartMediaInsightsQueuePolicy",
Queues=[Ref(start_media_insights_queue)],
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["sqs:SendMessage"],
"Resource": GetAtt(start_media_insights_queue, 'Arn'),
"Principal": {
"Service": "sns.amazonaws.com"
},
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(request_encoding_topic)
}
},
}],
},
))
template.add_resource(
TopicPolicy(
'UploadTopicPolicy',
Topics=[Ref(upload_topic)],
"QueueArn",
Value=GetAtt(sqsqueue, "Arn"),
Description="ARN of SQS Queue",
))
t.add_resource(
QueuePolicy("AllowSNS2SQSPolicy",
Queues=[Ref(sqsqueue)],
PolicyDocument={
"Version":
"2008-10-17",
"Id":
"PublicationPolicy",
"Statement": [{
"Sid": "Allow-SNS-SendMessage",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": ["sqs:SendMessage"],
"Resource": GetAtt(sqsqueue, "Arn"),
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(snstopic)
}
}
}]
}))
print(t.to_json())
)
],
))
template.add_resource(
QueuePolicy(
"RekognitionUpdatesQueuePolicy",
Queues=[Ref(rekognition_updates_queue)],
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["sqs:SendMessage"],
"Resource": GetAtt(rekognition_updates_queue, 'Arn'),
"Principal": {
"Service": "sns.amazonaws.com"
},
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(rekognition_updates_topic)
}
},
}],
},
))
rekognition_publish_role = template.add_resource(
Role(
'RekognitionPublishRole',
Path="/",
"JsonNotificationDLQ", "Arn"),
maxReceiveCount=3)))
t.add_resource(
Queue(
"JsonNotificationDLQ",
QueueName=Sub("${LambdaEnv}-json-notification-inbound-dlq"),
))
t.add_resource(
QueuePolicy(
"JsonNotificationReceiveQueuePolicy",
Queues=[Ref("JsonNotificationReceiveQueue")],
PolicyDocument=Policy(Statement=[
Statement(Effect=Allow,
Action=[SendMessage],
Resource=[GetAtt("JsonNotificationReceiveQueue", "Arn")],
Principal=AWSPrincipal("*"),
Condition=Condition([
ArnLike("aws:SourceArn", Ref("GalileoBabelTopicArn"))
]))
])))
t.add_resource(
Bucket("NotificationsToBeIngested",
BucketName=Sub("${LambdaEnv}-editorial-search-galileo-babel"),
DeletionPolicy="Retain",
NotificationConfiguration=NotificationConfiguration(
TopicConfigurations=[
TopicConfigurations(Event="s3:ObjectCreated:*",
Topic=ImportValue(
Sub("${LambdaEnv}-JsonTopicArn")))
def emit_configuration():
# Build an SQS queue for the babysitter
"""create_queue = template.add_parameter(
Parameter(
'CreateDeregistrationTopic',
Type='String',
Description='Whether or not to create the Chef Deregistration queue. This option is provided in case the queue already exists.',
Default='no',
AllowedValues=['yes', 'no'],
ConstraintDescription='Answer must be yes or no'
)
)
conditions = {
"CreateDeregCondition": Equals(
Ref(create_queue), "yes"
)
}
for c in conditions:
template.add_condition(c, conditions[c])"""
queue_name = '_'.join(['chef-deregistration', CLOUDNAME, CLOUDENV])
queue = template.add_resource(
Queue(
cfn.sanitize_id(queue_name),
VisibilityTimeout=60,
MessageRetentionPeriod=1209600,
MaximumMessageSize=16384,
QueueName=queue_name,
))
alert_topic = template.add_resource(
Topic(
cfn.sanitize_id("BabysitterAlarmTopic{0}".format(CLOUDENV)),
DisplayName='Babysitter Alarm',
TopicName=queue_name,
Subscription=[
Subscription(Endpoint=GetAtt(queue, "Arn"), Protocol='sqs'),
],
DependsOn=queue.title,
))
queue_depth_alarm = template.add_resource(
Alarm(
"BabysitterQueueDepthAlarm",
AlarmDescription=
'Alarm if the queue depth grows beyond 200 messages',
Namespace='AWS/SQS',
MetricName='ApproximateNumberOfMessagesVisible',
Dimensions=[
MetricDimension(Name='QueueName',
Value=GetAtt(queue, "QueueName"))
],
Statistic='Sum',
Period='300',
EvaluationPeriods='1',
Threshold='200',
ComparisonOperator='GreaterThanThreshold',
#AlarmActions=[Ref(alert_topic), ],
#InsufficientDataActions=[Ref(alert_topic), ],
DependsOn=alert_topic.title,
), )
queue_policy = {
"Version":
"2012-10-17",
"Id":
"BabysitterSNSPublicationPolicy",
"Statement": [{
"Sid": "AllowSNSPublishing",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": ["sqs:SendMessage"],
"Resource": GetAtt(queue, "Arn"),
"Condition": {
"ArnEquals": {
"aws:SourceArn": Ref(alert_topic)
}
}
}]
}
# Publish all events from SNS to the Queue
template.add_resource(
QueuePolicy(
"BabysitterPublishSNStoSQSPolicy",
Queues=[Ref(queue)],
PolicyDocument=queue_policy,
DependsOn=[queue.title, alert_topic.title],
))
cfn.alert_topic = alert_topic
def generate(env='pilot'):
template = Template()
template.set_version("2010-09-09")
# ExistingVPC = template.add_parameter(Parameter(
# "ExistingVPC",
# Type="AWS::EC2::VPC::Id",
# Description=(
# "The VPC ID that includes the security groups in the"
# "ExistingSecurityGroups parameter."
# ),
# ))
#
# ExistingSecurityGroups = template.add_parameter(Parameter(
# "ExistingSecurityGroups",
# Type="List<AWS::EC2::SecurityGroup::Id>",
# ))
param_spider_lambda_memory_size = template.add_parameter(
Parameter(
'SpiderLambdaMemorySize',
Type=NUMBER,
Description='Amount of memory to allocate to the Lambda Function',
Default='128',
AllowedValues=MEMORY_VALUES
)
)
param_spider_lambda_timeout = template.add_parameter(
Parameter(
'SpiderLambdaTimeout',
Type=NUMBER,
Description='Timeout in seconds for the Lambda function',
Default='60'
)
)
spider_tasks_queue_dlq_name = f'{env}-spider-tasks-dlq'
spider_tasks_queue_dlq = template.add_resource(
Queue(
"SpiderTasksDLQ",
QueueName=spider_tasks_queue_dlq_name,
MessageRetentionPeriod=(60 * 60 * 24 * 14),
)
)
spider_tasks_queue_name = f"{env}-spider-tasks"
spider_tasks_queue = template.add_resource(
Queue(
"SpiderTasksQueue",
QueueName=spider_tasks_queue_name,
MessageRetentionPeriod=(60 * 60 * 24 * 14),
VisibilityTimeout=300,
RedrivePolicy=RedrivePolicy(
deadLetterTargetArn=GetAtt(spider_tasks_queue_dlq, "Arn"),
maxReceiveCount=2,
),
DependsOn=[spider_tasks_queue_dlq],
)
)
spider_lambda_role = template.add_resource(
Role(
"SpiderLambdaRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="root",
Statement=[
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("logs", "*")
]
),
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("s3", "*")
]
),
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("sqs", "*")
]
),
]
),
)
],
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
}
}]
},
)
)
spider_file_path = './spider/index.js'
spider_code = open(spider_file_path, 'r').readlines()
spider_lambda = template.add_resource(
Function(
"SpiderLambda",
Code=Code(
S3Bucket='spider-lambda',
S3Key=f'{env}.zip',
# ZipFile=Join("", spider_code)
),
Handler="index.handler",
Role=GetAtt(spider_lambda_role, "Arn"),
Runtime="nodejs12.x",
Layers=['arn:aws:lambda:us-east-1:342904801388:layer:spider-node-browser:1'],
MemorySize=Ref(param_spider_lambda_memory_size),
Timeout=Ref(param_spider_lambda_timeout),
DependsOn=[spider_tasks_queue],
)
)
# AllSecurityGroups = template.add_resource(CustomResource(
# "AllSecurityGroups",
# List=Ref(ExistingSecurityGroups),
# AppendedItem=Ref("SecurityGroup"),
# ServiceToken=GetAtt(spider_lambda, "Arn"),
# ))
#
# SecurityGroup = template.add_resource(SecurityGroup(
# "SecurityGroup",
# SecurityGroupIngress=[
# {"ToPort": "80", "IpProtocol": "tcp", "CidrIp": "0.0.0.0/0",
# "FromPort": "80"}],
# VpcId=Ref(ExistingVPC),
# GroupDescription="Allow HTTP traffic to the host",
# SecurityGroupEgress=[
# {"ToPort": "80", "IpProtocol": "tcp", "CidrIp": "0.0.0.0/0",
# "FromPort": "80"}],
# ))
#
# AllSecurityGroups = template.add_output(Output(
# "AllSecurityGroups",
# Description="Security Groups that are associated with the EC2 instance",
# Value=Join(", ", GetAtt(AllSecurityGroups, "Value")),
# ))
source_sns_name = f'{env}-source-sns-topic'
source_sns_topic = template.add_resource(
Topic(
"SNSSource",
TopicName=source_sns_name,
Subscription=[
Subscription(
Endpoint=GetAtt(spider_tasks_queue, "Arn"),
Protocol='sqs',
)
],
DependsOn=[spider_tasks_queue]
)
)
source_sns_topic_policy = template.add_resource(
TopicPolicy(
"SourceForwardingTopicPolicy",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="AllowS3PutMessageInSNS",
Statement=[
Statement(
Sid="AllowS3PutMessages",
Principal=Principal("Service", "s3.amazonaws.com"),
Effect=Allow,
Action=[
Action("sns", "Publish"),
],
Resource=["*"],
)
]
),
Topics=[Ref(source_sns_topic)],
)
)
sns_sqs_policy = template.add_resource(
QueuePolicy(
"AllowSNSPutMessagesInSQS",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="AllowSNSPutMessagesInSQS",
Statement=[
Statement(
Sid="AllowSNSPutMessagesInSQS2",
Principal=Principal("*"),
Effect=Allow,
Action=[
Action("sqs", "SendMessage"),
],
Resource=["*"],
)
]
),
Queues=[Ref(spider_tasks_queue)],
DependsOn=[spider_tasks_queue],
)
)
# Buckets
source_bucket_name = f'{env}-source-bucket'
source_bucket = template.add_resource(
Bucket(
"SourceBucket",
BucketName=source_bucket_name,
NotificationConfiguration=NotificationConfiguration(
TopicConfigurations=[
TopicConfigurations(
Topic=Ref(source_sns_topic),
Event="s3:ObjectCreated:*",
)
],
),
DependsOn=[source_sns_topic_policy],
)
)
results_bucket_name = f'{env}-results-bucket'
results_bucket = template.add_resource(
Bucket(
"ResultsBucket",
BucketName=results_bucket_name,
)
)
# Lambda trigger
template.add_resource(
EventSourceMapping(
"TriggerLambdaSpiderFromSQS",
EventSourceArn=GetAtt(spider_tasks_queue, "Arn"),
FunctionName=Ref(spider_lambda),
BatchSize=1, # Default process tasks one by one
)
)
return template.to_json()
Description="Name of Captured Data SQS",
Default="simple-queue",
Type="String"))
sqsqueue = template.add_resource(
Queue("CapturedDataQueue", QueueName=Ref("SQSQueueName")))
template.add_resource(
QueuePolicy(
"CapturedDataQueuePolicy",
Queues=[Ref(sqsqueue)],
PolicyDocument={
"Version":
"2012-10-17",
"Statement": [{
"Sid": "LambdaWriteToQueue",
"Effect": "Allow",
"Principal": {
"AWS": GetAtt("LambdaExecutionRole", "Arn")
},
"Action": "SQS:*",
"Resource": GetAtt(sqsqueue, "Arn")
}]
}))
kinesis_stream = template.add_resource(
kinesis.Stream(
"CapturedDataKinesisStream",
Name=Ref(kinesis_param),
ShardCount=1,
RetentionPeriodHours=24))