Environment=Environment(
Variables={
'VIDEO_EVENTS_TABLE':
ImportValue(
Join('-', [Ref(core_stack), 'VideoEventsTable', 'Ref'])),
'EVENT_BUS_NAME':
Ref(event_bus),
}),
TracingConfig=TracingConfig(Mode='Active', ),
))
template.add_resource(
LogGroup(
"StartInsightsLogGroup",
LogGroupName=Join(
'/', ['/aws/lambda', Ref(start_insights_function)]),
RetentionInDays=7,
))
template.add_resource(
EventSourceMapping(
'InvokeStartInsights',
EventSourceArn=ImportValue(
Join('-', [Ref(encoding_stack), 'StartMediaInsightsQueue-Arn'])),
FunctionName=Ref(start_insights_function),
Enabled=True,
))
f = open("output/video_insights_engine.json", "w")
f.write(template.to_json())
def _add_adaptive_memory_functions(self, template, context: ff.Context,
timeout, role_title, async_lambda):
memory_settings = self._configuration.contexts.get('firefly_aws').get(
'memory_settings')
if memory_settings is None:
raise ff.ConfigurationError(
'To use adaptive memory, you must provide a list of memory_settings'
)
for memory in memory_settings:
memory_size = template.add_parameter(
Parameter(
f'{self._lambda_resource_name(context.name)}{memory}MemorySize',
Type=NUMBER,
Default=str(memory)))
params = {
'FunctionName':
self._lambda_function_name(context.name,
'Async',
memory=memory),
'Role':
GetAtt(role_title, 'Arn'),
'MemorySize':
Ref(memory_size),
'Timeout':
Ref(timeout),
'Environment':
self._lambda_environment(context),
'DependsOn':
async_lambda,
}
image_uri = self._aws_config.get('image_uri')
if image_uri is not None:
params.update({
'Code': Code(ImageUri=image_uri),
'PackageType': 'Image',
})
else:
params.update({
'Code':
Code(S3Bucket=self._bucket, S3Key=self._code_key),
'Runtime':
'python3.7',
'Handler':
'handlers.main',
})
if self._security_group_ids and self._subnet_ids:
params['VpcConfig'] = VPCConfig(
SecurityGroupIds=self._security_group_ids,
SubnetIds=self._subnet_ids)
adaptive_memory_lambda = template.add_resource(
Function(
self._lambda_resource_name(context.name, memory=memory),
**params))
queue = template.add_resource(
Queue(self._queue_name(context.name, memory=memory),
QueueName=self._queue_name(context.name, memory=memory),
VisibilityTimeout=905,
ReceiveMessageWaitTimeSeconds=20,
MessageRetentionPeriod=1209600,
DependsOn=[adaptive_memory_lambda]))
# self._queue_policy(template, queue, self._queue_name(context.name), subscriptions)
template.add_resource(
EventSourceMapping(
f'{self._lambda_resource_name(context.name, memory=memory)}AsyncMapping',
BatchSize=1,
Enabled=True,
EventSourceArn=GetAtt(queue, 'Arn'),
FunctionName=self._lambda_function_name(context.name,
'Async',
memory=memory),
DependsOn=[queue, adaptive_memory_lambda]))
Resource=[
Join("/", [
Join("", [
"arn:aws:s3:::",
Sub("${LambdaEnv}"),
"-editorial-search-galileo-babel"
]), "*"
])
],
)
])))
t.add_resource(
EventSourceMapping("FunctionEventSourceMapping",
EventSourceArn=GetAtt("JsonNotificationReceiveQueue",
"Arn"),
FunctionName=Ref("LambdaFunction"),
BatchSize=1))
t.add_resource(
Permission("InvokeLambdaPermission",
FunctionName=Join(
':', [GetAtt("LambdaFunction", "Arn"),
Ref("LambdaEnv")]),
Action="lambda:InvokeFunction",
SourceArn=GetAtt("JsonNotificationReceiveQueue", "Arn"),
Principal="sqs.amazonaws.com"))
t.add_resource(
Alias("GalileoBabelLambdaAlias",
Description="Alias for the galileo babel lambda",
def _deploy_service(self, service: ff.Service):
context = self._context_map.get_context(service.name)
if self._aws_config.get('image_uri') is None:
self._package_and_deploy_code(context)
template = Template()
template.set_version('2010-09-09')
memory_size = template.add_parameter(
Parameter(f'{self._lambda_resource_name(service.name)}MemorySize',
Type=NUMBER,
Default=self._aws_config.get('memory_sync', '3008')))
timeout_gateway = template.add_parameter(
Parameter(
f'{self._lambda_resource_name(service.name)}GatewayTimeout',
Type=NUMBER,
Default='30'))
timeout_async = template.add_parameter(
Parameter(
f'{self._lambda_resource_name(service.name)}AsyncTimeout',
Type=NUMBER,
Default='900'))
role_title = f'{self._lambda_resource_name(service.name)}ExecutionRole'
role = self._add_role(role_title, template)
params = {
'FunctionName': f'{self._service_name(service.name)}Sync',
'Role': GetAtt(role_title, 'Arn'),
'MemorySize': Ref(memory_size),
'Timeout': Ref(timeout_gateway),
'Environment': self._lambda_environment(context)
}
image_uri = self._aws_config.get('image_uri')
if image_uri is not None:
params.update({
'Code': Code(ImageUri=image_uri),
'PackageType': 'Image',
})
else:
params.update({
'Code':
Code(S3Bucket=self._bucket, S3Key=self._code_key),
'Runtime':
'python3.7',
'Handler':
'handlers.main',
})
if self._security_group_ids and self._subnet_ids:
params['VpcConfig'] = VPCConfig(
SecurityGroupIds=self._security_group_ids,
SubnetIds=self._subnet_ids)
api_lambda = template.add_resource(
Function(f'{self._lambda_resource_name(service.name)}Sync',
**params))
route = inflection.dasherize(context.name)
proxy_route = f'{route}/{{proxy+}}'
template.add_resource(
Permission(
f'{self._lambda_resource_name(service.name)}SyncPermission',
Action='lambda:InvokeFunction',
FunctionName=f'{self._service_name(service.name)}Sync',
Principal='apigateway.amazonaws.com',
SourceArn=Join('', [
'arn:aws:execute-api:', self._region, ':',
self._account_id, ':',
ImportValue(
self._rest_api_reference()), '/*/*/', route, '*'
]),
DependsOn=api_lambda))
if self._adaptive_memory:
value = '3008' if not self._adaptive_memory else '256'
try:
value = int(self._aws_config.get('memory_async'))
except ValueError:
pass
memory_size = template.add_parameter(
Parameter(
f'{self._lambda_resource_name(service.name)}MemorySizeAsync',
Type=NUMBER,
Default=value))
params = {
'FunctionName': self._lambda_function_name(service.name, 'Async'),
'Role': GetAtt(role_title, 'Arn'),
'MemorySize': Ref(memory_size),
'Timeout': Ref(timeout_async),
'Environment': self._lambda_environment(context)
}
if image_uri is not None:
params.update({
'Code': Code(ImageUri=image_uri),
'PackageType': 'Image',
})
else:
params.update({
'Code':
Code(S3Bucket=self._bucket, S3Key=self._code_key),
'Runtime':
'python3.7',
'Handler':
'handlers.main',
})
if self._security_group_ids and self._subnet_ids:
params['VpcConfig'] = VPCConfig(
SecurityGroupIds=self._security_group_ids,
SubnetIds=self._subnet_ids)
async_lambda = template.add_resource(
Function(self._lambda_resource_name(service.name, type_='Async'),
**params))
if self._adaptive_memory:
self._add_adaptive_memory_functions(template, context,
timeout_async, role_title,
async_lambda)
# self._add_adaptive_memory_streams(template, context, async_lambda, role)
# Timers
for cls, _ in context.command_handlers.items():
if cls.has_timer():
timer = cls.get_timer()
if timer.environment is not None and timer.environment != self._env:
continue
if isinstance(timer.command, str):
timer_name = timer.command
else:
timer_name = timer.command.__name__
target = Target(
f'{self._service_name(service.name)}AsyncTarget',
Arn=GetAtt(
self._lambda_resource_name(service.name,
type_='Async'), 'Arn'),
Id=self._lambda_resource_name(service.name, type_='Async'),
Input=
f'{{"_context": "{context.name}", "_type": "command", "_name": "{cls.__name__}"}}'
)
rule = template.add_resource(
Rule(f'{timer_name}TimerRule',
ScheduleExpression=f'cron({timer.cron})',
State='ENABLED',
Targets=[target]))
template.add_resource(
Permission(f'{timer_name}TimerPermission',
Action='lambda:invokeFunction',
Principal='events.amazonaws.com',
FunctionName=Ref(async_lambda),
SourceArn=GetAtt(rule, 'Arn')))
integration = template.add_resource(
Integration(
self._integration_name(context.name),
ApiId=ImportValue(self._rest_api_reference()),
PayloadFormatVersion='2.0',
IntegrationType='AWS_PROXY',
IntegrationUri=Join('', [
'arn:aws:lambda:',
self._region,
':',
self._account_id,
':function:',
Ref(api_lambda),
]),
))
template.add_resource(
Route(f'{self._route_name(context.name)}Base',
ApiId=ImportValue(self._rest_api_reference()),
RouteKey=f'ANY /{route}',
AuthorizationType='NONE',
Target=Join(
'/', ['integrations', Ref(integration)]),
DependsOn=integration))
template.add_resource(
Route(f'{self._route_name(context.name)}Proxy',
ApiId=ImportValue(self._rest_api_reference()),
RouteKey=f'ANY /{proxy_route}',
AuthorizationType='NONE',
Target=Join(
'/', ['integrations', Ref(integration)]),
DependsOn=integration))
# Error alarms / subscriptions
if 'errors' in self._aws_config:
alerts_topic = template.add_resource(
Topic(self._alert_topic_name(service.name),
TopicName=self._alert_topic_name(service.name)))
if 'email' in self._aws_config.get('errors'):
for address in self._aws_config.get('errors').get('email').get(
'recipients').split(','):
template.add_resource(
SubscriptionResource(
self._alarm_subscription_name(context.name),
Protocol='email',
Endpoint=address,
TopicArn=self._alert_topic_arn(context.name),
DependsOn=[alerts_topic]))
# Queues / Topics
subscriptions = {}
for subscription in self._get_subscriptions(context):
if subscription['context'] not in subscriptions:
subscriptions[subscription['context']] = []
subscriptions[subscription['context']].append(subscription)
dlq = template.add_resource(
Queue(f'{self._queue_name(context.name)}Dlq',
QueueName=f'{self._queue_name(context.name)}Dlq',
VisibilityTimeout=905,
ReceiveMessageWaitTimeSeconds=20,
MessageRetentionPeriod=1209600))
self._queue_policy(template, dlq,
f'{self._queue_name(context.name)}Dlq',
subscriptions)
queue = template.add_resource(
Queue(self._queue_name(context.name),
QueueName=self._queue_name(context.name),
VisibilityTimeout=905,
ReceiveMessageWaitTimeSeconds=20,
MessageRetentionPeriod=1209600,
RedrivePolicy=RedrivePolicy(deadLetterTargetArn=GetAtt(
dlq, 'Arn'),
maxReceiveCount=1000),
DependsOn=dlq))
self._queue_policy(template, queue, self._queue_name(context.name),
subscriptions)
template.add_resource(
EventSourceMapping(
f'{self._lambda_resource_name(context.name)}AsyncMapping',
BatchSize=1,
Enabled=True,
EventSourceArn=GetAtt(queue, 'Arn'),
FunctionName=self._lambda_function_name(service.name, 'Async'),
DependsOn=[queue, async_lambda]))
topic = template.add_resource(
Topic(self._topic_name(context.name),
TopicName=self._topic_name(context.name)))
for context_name, list_ in subscriptions.items():
if context_name == context.name and len(list_) > 0:
template.add_resource(
SubscriptionResource(
self._subscription_name(context_name),
Protocol='sqs',
Endpoint=GetAtt(queue, 'Arn'),
TopicArn=self._topic_arn(context.name),
FilterPolicy={
'_name': [x['name'] for x in list_],
},
RedrivePolicy={
'deadLetterTargetArn': GetAtt(dlq, 'Arn'),
},
DependsOn=[queue, dlq, topic]))
elif len(list_) > 0:
if context_name not in self._context_map.contexts:
self._find_or_create_topic(context_name)
template.add_resource(
SubscriptionResource(
self._subscription_name(context.name, context_name),
Protocol='sqs',
Endpoint=GetAtt(queue, 'Arn'),
TopicArn=self._topic_arn(context_name),
FilterPolicy={'_name': [x['name'] for x in list_]},
RedrivePolicy={
'deadLetterTargetArn': GetAtt(dlq, 'Arn'),
},
DependsOn=[queue, dlq]))
# DynamoDB Table
ddb_table = template.add_resource(
Table(self._ddb_resource_name(context.name),
TableName=self._ddb_table_name(context.name),
AttributeDefinitions=[
AttributeDefinition(AttributeName='pk',
AttributeType='S'),
AttributeDefinition(AttributeName='sk',
AttributeType='S'),
],
BillingMode='PAY_PER_REQUEST',
KeySchema=[
KeySchema(AttributeName='pk', KeyType='HASH'),
KeySchema(AttributeName='sk', KeyType='RANGE'),
],
TimeToLiveSpecification=TimeToLiveSpecification(
AttributeName='TimeToLive', Enabled=True)))
template.add_output(
Output("DDBTable",
Value=Ref(ddb_table),
Description="Document table"))
for cb in self._pre_deployment_hooks:
cb(template=template, context=context, env=self._env)
self.info('Deploying stack')
self._s3_client.put_object(Body=template.to_json(),
Bucket=self._bucket,
Key=self._template_key)
url = self._s3_client.generate_presigned_url(ClientMethod='get_object',
Params={
'Bucket':
self._bucket,
'Key':
self._template_key
})
stack_name = self._stack_name(context.name)
try:
self._cloudformation_client.describe_stacks(StackName=stack_name)
self._update_stack(self._stack_name(context.name), url)
except ClientError as e:
if f'Stack with id {stack_name} does not exist' in str(e):
self._create_stack(self._stack_name(context.name), url)
else:
raise e
for cb in self._post_deployment_hooks:
cb(template=template, context=context, env=self._env)
self._migrate_schema(context)
self.info('Done')
def _deploy_service(self, service: ff.Service):
context = self._context_map.get_context(service.name)
self._package_and_deploy_code(context)
template = Template()
template.set_version('2010-09-09')
memory_size = template.add_parameter(Parameter(
f'{self._lambda_resource_name(service.name)}MemorySize',
Type=NUMBER,
Default='3008'
))
timeout_gateway = template.add_parameter(Parameter(
f'{self._lambda_resource_name(service.name)}GatewayTimeout',
Type=NUMBER,
Default='30'
))
timeout_async = template.add_parameter(Parameter(
f'{self._lambda_resource_name(service.name)}AsyncTimeout',
Type=NUMBER,
Default='900'
))
role_title = f'{self._lambda_resource_name(service.name)}ExecutionRole'
self._add_role(role_title, template)
params = {
'FunctionName': f'{self._service_name(service.name)}Sync',
'Code': Code(
S3Bucket=self._bucket,
S3Key=self._code_key
),
'Handler': 'handlers.main',
'Role': GetAtt(role_title, 'Arn'),
'Runtime': 'python3.7',
'MemorySize': Ref(memory_size),
'Timeout': Ref(timeout_gateway),
'Environment': self._lambda_environment(context)
}
if self._security_group_ids and self._subnet_ids:
params['VpcConfig'] = VPCConfig(
SecurityGroupIds=self._security_group_ids,
SubnetIds=self._subnet_ids
)
api_lambda = template.add_resource(Function(
f'{self._lambda_resource_name(service.name)}Sync',
**params
))
route = inflection.dasherize(context.name)
proxy_route = f'{route}/{{proxy+}}'
template.add_resource(Permission(
f'{self._lambda_resource_name(service.name)}SyncPermission',
Action='lambda:InvokeFunction',
FunctionName=f'{self._service_name(service.name)}Sync',
Principal='apigateway.amazonaws.com',
SourceArn=Join('', [
'arn:aws:execute-api:',
self._region,
':',
self._account_id,
':',
ImportValue(self._rest_api_reference()),
'/*/*/',
route,
'*'
]),
DependsOn=api_lambda
))
params = {
'FunctionName': f'{self._service_name(service.name)}Async',
'Code': Code(
S3Bucket=self._bucket,
S3Key=self._code_key
),
'Handler': 'handlers.main',
'Role': GetAtt(role_title, 'Arn'),
'Runtime': 'python3.7',
'MemorySize': Ref(memory_size),
'Timeout': Ref(timeout_async),
'Environment': self._lambda_environment(context)
}
if self._security_group_ids and self._subnet_ids:
params['VpcConfig'] = VPCConfig(
SecurityGroupIds=self._security_group_ids,
SubnetIds=self._subnet_ids
)
async_lambda = template.add_resource(Function(
f'{self._lambda_resource_name(service.name)}Async',
**params
))
integration = template.add_resource(Integration(
self._integration_name(context.name),
ApiId=ImportValue(self._rest_api_reference()),
PayloadFormatVersion='2.0',
IntegrationType='AWS_PROXY',
IntegrationUri=Join('', [
'arn:aws:lambda:',
self._region,
':',
self._account_id,
':function:',
Ref(api_lambda),
]),
))
template.add_resource(Route(
f'{self._route_name(context.name)}Base',
ApiId=ImportValue(self._rest_api_reference()),
RouteKey=f'ANY /{route}',
AuthorizationType='NONE',
Target=Join('/', ['integrations', Ref(integration)]),
DependsOn=integration
))
template.add_resource(Route(
f'{self._route_name(context.name)}Proxy',
ApiId=ImportValue(self._rest_api_reference()),
RouteKey=f'ANY /{proxy_route}',
AuthorizationType='NONE',
Target=Join('/', ['integrations', Ref(integration)]),
DependsOn=integration
))
# Error alarms / subscriptions
if 'errors' in self._aws_config:
alerts_topic = template.add_resource(Topic(
self._alert_topic_name(service.name),
TopicName=self._alert_topic_name(service.name)
))
self._add_error_alarm(template, f'{self._service_name(context.name)}Sync', context.name, alerts_topic)
self._add_error_alarm(template, f'{self._service_name(context.name)}Async', context.name, alerts_topic)
if 'email' in self._aws_config.get('errors'):
template.add_resource(SubscriptionResource(
self._alarm_subscription_name(context.name),
Protocol='email',
Endpoint=self._aws_config.get('errors').get('email').get('recipients'),
TopicArn=self._alert_topic_arn(context.name),
DependsOn=[alerts_topic]
))
# Queues / Topics
subscriptions = {}
for subscription in self._get_subscriptions(context):
if subscription['context'] not in subscriptions:
subscriptions[subscription['context']] = []
subscriptions[subscription['context']].append(subscription)
dlq = template.add_resource(Queue(
f'{self._queue_name(context.name)}Dlq',
QueueName=f'{self._queue_name(context.name)}Dlq',
VisibilityTimeout=905,
ReceiveMessageWaitTimeSeconds=20,
MessageRetentionPeriod=1209600
))
self._queue_policy(template, dlq, f'{self._queue_name(context.name)}Dlq', subscriptions)
queue = template.add_resource(Queue(
self._queue_name(context.name),
QueueName=self._queue_name(context.name),
VisibilityTimeout=905,
ReceiveMessageWaitTimeSeconds=20,
MessageRetentionPeriod=1209600,
RedrivePolicy=RedrivePolicy(
deadLetterTargetArn=GetAtt(dlq, 'Arn'),
maxReceiveCount=1000
),
DependsOn=dlq
))
self._queue_policy(template, queue, self._queue_name(context.name), subscriptions)
template.add_resource(EventSourceMapping(
f'{self._lambda_resource_name(context.name)}AsyncMapping',
BatchSize=1,
Enabled=True,
EventSourceArn=GetAtt(queue, 'Arn'),
FunctionName=f'{self._service_name(service.name)}Async',
DependsOn=[queue, async_lambda]
))
topic = template.add_resource(Topic(
self._topic_name(context.name),
TopicName=self._topic_name(context.name)
))
for context_name, list_ in subscriptions.items():
if context_name == context.name and len(list_) > 0:
template.add_resource(SubscriptionResource(
self._subscription_name(context_name),
Protocol='sqs',
Endpoint=GetAtt(queue, 'Arn'),
TopicArn=self._topic_arn(context.name),
FilterPolicy={
'_name': [x['name'] for x in list_],
},
RedrivePolicy={
'deadLetterTargetArn': GetAtt(dlq, 'Arn'),
},
DependsOn=[queue, dlq, topic]
))
elif len(list_) > 0:
if context_name not in self._context_map.contexts:
self._find_or_create_topic(context_name)
template.add_resource(SubscriptionResource(
self._subscription_name(context.name, context_name),
Protocol='sqs',
Endpoint=GetAtt(queue, 'Arn'),
TopicArn=self._topic_arn(context_name),
FilterPolicy={
'_name': [x['name'] for x in list_]
},
RedrivePolicy={
'deadLetterTargetArn': GetAtt(dlq, 'Arn'),
},
DependsOn=[queue, dlq]
))
self.info('Deploying stack')
stack_name = self._stack_name(context.name)
try:
self._cloudformation_client.describe_stacks(StackName=stack_name)
self._update_stack(self._stack_name(context.name), template)
except ClientError as e:
if f'Stack with id {stack_name} does not exist' in str(e):
self._create_stack(self._stack_name(context.name), template)
else:
raise e
self._execute_ddl(context)
self.info('Done')
}
),
TracingConfig=TracingConfig(
Mode='Active',
),
))
template.add_resource(LogGroup(
"ConsumeEventsLogGroup",
LogGroupName=Join('/', ['/aws/lambda', Ref(consume_events_function)]),
RetentionInDays=7,
))
template.add_resource(EventSourceMapping(
'InvokeConsumeEvents',
EventSourceArn=ImportValue(Join('-', [Ref(core_stack), 'EventsToApiQueue-Arn'])),
FunctionName=Ref(consume_events_function),
Enabled=True,
))
cloudfront_certificate = template.add_resource(Certificate(
"CloudFrontCertificate",
DomainName=Ref(domain_name),
DomainValidationOptions=[DomainValidationOption(
DomainName=Ref(domain_name),
ValidationDomain=ImportValue(Join('-', [Ref(dns_stack), 'HostedZoneName'])),
)],
ValidationMethod='DNS',
))
api_gateway = template.add_resource(RestApi(
'ApiGateway',
def generate(env='pilot'):
template = Template()
template.set_version("2010-09-09")
# ExistingVPC = template.add_parameter(Parameter(
# "ExistingVPC",
# Type="AWS::EC2::VPC::Id",
# Description=(
# "The VPC ID that includes the security groups in the"
# "ExistingSecurityGroups parameter."
# ),
# ))
#
# ExistingSecurityGroups = template.add_parameter(Parameter(
# "ExistingSecurityGroups",
# Type="List<AWS::EC2::SecurityGroup::Id>",
# ))
param_spider_lambda_memory_size = template.add_parameter(
Parameter(
'SpiderLambdaMemorySize',
Type=NUMBER,
Description='Amount of memory to allocate to the Lambda Function',
Default='128',
AllowedValues=MEMORY_VALUES
)
)
param_spider_lambda_timeout = template.add_parameter(
Parameter(
'SpiderLambdaTimeout',
Type=NUMBER,
Description='Timeout in seconds for the Lambda function',
Default='60'
)
)
spider_tasks_queue_dlq_name = f'{env}-spider-tasks-dlq'
spider_tasks_queue_dlq = template.add_resource(
Queue(
"SpiderTasksDLQ",
QueueName=spider_tasks_queue_dlq_name,
MessageRetentionPeriod=(60 * 60 * 24 * 14),
)
)
spider_tasks_queue_name = f"{env}-spider-tasks"
spider_tasks_queue = template.add_resource(
Queue(
"SpiderTasksQueue",
QueueName=spider_tasks_queue_name,
MessageRetentionPeriod=(60 * 60 * 24 * 14),
VisibilityTimeout=300,
RedrivePolicy=RedrivePolicy(
deadLetterTargetArn=GetAtt(spider_tasks_queue_dlq, "Arn"),
maxReceiveCount=2,
),
DependsOn=[spider_tasks_queue_dlq],
)
)
spider_lambda_role = template.add_resource(
Role(
"SpiderLambdaRole",
Path="/",
Policies=[
Policy(
PolicyName="root",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="root",
Statement=[
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("logs", "*")
]
),
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("s3", "*")
]
),
Statement(
Effect=Allow,
Resource=["*"],
Action=[
Action("sqs", "*")
]
),
]
),
)
],
AssumeRolePolicyDocument={
"Version": "2012-10-17",
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
}
}]
},
)
)
spider_file_path = './spider/index.js'
spider_code = open(spider_file_path, 'r').readlines()
spider_lambda = template.add_resource(
Function(
"SpiderLambda",
Code=Code(
S3Bucket='spider-lambda',
S3Key=f'{env}.zip',
# ZipFile=Join("", spider_code)
),
Handler="index.handler",
Role=GetAtt(spider_lambda_role, "Arn"),
Runtime="nodejs12.x",
Layers=['arn:aws:lambda:us-east-1:342904801388:layer:spider-node-browser:1'],
MemorySize=Ref(param_spider_lambda_memory_size),
Timeout=Ref(param_spider_lambda_timeout),
DependsOn=[spider_tasks_queue],
)
)
# AllSecurityGroups = template.add_resource(CustomResource(
# "AllSecurityGroups",
# List=Ref(ExistingSecurityGroups),
# AppendedItem=Ref("SecurityGroup"),
# ServiceToken=GetAtt(spider_lambda, "Arn"),
# ))
#
# SecurityGroup = template.add_resource(SecurityGroup(
# "SecurityGroup",
# SecurityGroupIngress=[
# {"ToPort": "80", "IpProtocol": "tcp", "CidrIp": "0.0.0.0/0",
# "FromPort": "80"}],
# VpcId=Ref(ExistingVPC),
# GroupDescription="Allow HTTP traffic to the host",
# SecurityGroupEgress=[
# {"ToPort": "80", "IpProtocol": "tcp", "CidrIp": "0.0.0.0/0",
# "FromPort": "80"}],
# ))
#
# AllSecurityGroups = template.add_output(Output(
# "AllSecurityGroups",
# Description="Security Groups that are associated with the EC2 instance",
# Value=Join(", ", GetAtt(AllSecurityGroups, "Value")),
# ))
source_sns_name = f'{env}-source-sns-topic'
source_sns_topic = template.add_resource(
Topic(
"SNSSource",
TopicName=source_sns_name,
Subscription=[
Subscription(
Endpoint=GetAtt(spider_tasks_queue, "Arn"),
Protocol='sqs',
)
],
DependsOn=[spider_tasks_queue]
)
)
source_sns_topic_policy = template.add_resource(
TopicPolicy(
"SourceForwardingTopicPolicy",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="AllowS3PutMessageInSNS",
Statement=[
Statement(
Sid="AllowS3PutMessages",
Principal=Principal("Service", "s3.amazonaws.com"),
Effect=Allow,
Action=[
Action("sns", "Publish"),
],
Resource=["*"],
)
]
),
Topics=[Ref(source_sns_topic)],
)
)
sns_sqs_policy = template.add_resource(
QueuePolicy(
"AllowSNSPutMessagesInSQS",
PolicyDocument=PolicyDocument(
Version="2012-10-17",
Id="AllowSNSPutMessagesInSQS",
Statement=[
Statement(
Sid="AllowSNSPutMessagesInSQS2",
Principal=Principal("*"),
Effect=Allow,
Action=[
Action("sqs", "SendMessage"),
],
Resource=["*"],
)
]
),
Queues=[Ref(spider_tasks_queue)],
DependsOn=[spider_tasks_queue],
)
)
# Buckets
source_bucket_name = f'{env}-source-bucket'
source_bucket = template.add_resource(
Bucket(
"SourceBucket",
BucketName=source_bucket_name,
NotificationConfiguration=NotificationConfiguration(
TopicConfigurations=[
TopicConfigurations(
Topic=Ref(source_sns_topic),
Event="s3:ObjectCreated:*",
)
],
),
DependsOn=[source_sns_topic_policy],
)
)
results_bucket_name = f'{env}-results-bucket'
results_bucket = template.add_resource(
Bucket(
"ResultsBucket",
BucketName=results_bucket_name,
)
)
# Lambda trigger
template.add_resource(
EventSourceMapping(
"TriggerLambdaSpiderFromSQS",
EventSourceArn=GetAtt(spider_tasks_queue, "Arn"),
FunctionName=Ref(spider_lambda),
BatchSize=1, # Default process tasks one by one
)
)
return template.to_json()
"Statement": [{
"Action": ["sts:AssumeRole"],
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
}
}]
},
ManagedPolicyArns=[
"arn:aws:iam::aws:policy/AmazonKinesisReadOnlyAccess",
"arn:aws:iam::aws:policy/service-role/AWSLambdaRole"
]))
template.add_resource(
EventSourceMapping(
"LambdaKinesisTrigger",
BatchSize=100,
Enabled=True,
EventSourceArn=GetAtt(kinesis_stream, "Arn"),
FunctionName=GetAtt(kinesis_consumer, "Arn"),
StartingPosition="TRIM_HORIZON"))
template.add_output([
Output(
"CapturedDataStreamName",
Description="CapturedDataKinesisStreamName",
Value=Ref(kinesis_stream), ),
])
print(template.to_json())