def subnet_id_lookup(bosslet_config):
# DP NOTE: During implementation there was/is an availability zone that
# could not run the T2.Micro instances used by this Data Pipeline
azs = aws.azs_lookup(bosslet_config, 'datapipeline')
az = random.choice(azs)[1] + '-'
internal_subnet = aws.subnet_id_lookup(bosslet_config.session,
az + bosslet_config.names.internal.subnet)
return internal_subnet
def create_config(session, domain, keypair=None, user_data=None, db_config={}):
"""Create the CloudFormationConfiguration object."""
names = AWSNames(domain)
config = CloudFormationConfiguration('proofreader', domain, const.REGION)
vpc_id = config.find_vpc(session)
az_subnets, _ = config.find_all_availability_zones(session)
external_subnet_id = aws.subnet_id_lookup(session,
names.subnet("external"))
config.add_arg(
Arg.Subnet("ExternalSubnet", external_subnet_id,
"ID of External Subnet to create resources in"))
sgs = aws.sg_lookup_all(session, vpc_id)
# Only allow unsecure web access from APL, until a ELB is configured for HTTPS
config.add_security_group("HttpSecurityGroup", names.http,
[("tcp", "80", "80", const.INCOMING_SUBNET)])
config.add_ec2_instance("ProofreaderWeb",
names.proofreader,
aws.ami_lookup(session, "proofreader-web.boss"),
keypair,
public_ip=True,
subnet=Ref("ExternalSubnet"),
security_groups=[
sgs[names.internal], sgs[names.ssh],
Ref('HttpSecurityGroup')
],
user_data=user_data,
depends_on="ProofreaderDB"
) # make sure the DB is launched before we start
config.add_rds_db("ProofreaderDB",
names.proofreader_db,
db_config.get("port"),
db_config.get("name"),
db_config.get("user"),
db_config.get("password"),
az_subnets,
security_groups=[sgs[names.internal]])
return config
def create_config(bosslet_config):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('backup', bosslet_config)
names = bosslet_config.names
# DP NOTE: During implementation there was/is an availability zone that
# could not run the T2.Micro instances used by this Data Pipeline
azs = aws.azs_lookup(bosslet_config, 'datapipeline')
az = random.choice(azs)[1] + '-'
internal_subnet = aws.subnet_id_lookup(
bosslet_config.session, az + bosslet_config.names.internal.subnet)
backup_image = aws.ami_lookup(bosslet_config, names.backup.ami)[0]
s3_backup = "s3://" + names.backup.s3 + "/#{format(@scheduledStartTime, 'YYYY-ww')}"
s3_logs = "s3://" + names.backup.s3 + "/logs"
# DP TODO: Create all BOSS S3 buckets as part of the account setup
# as the Cloud Formation delete doesn't delete the bucket,
# making this a conditional add
BUCKET_DEPENDENCY = None # Needed as the pipelines try to execute when launched
if not aws.s3_bucket_exists(bosslet_config.session, names.backup.s3):
life_cycle = {
'Rules': [{
'Id': 'Delete Data',
'Status': 'Enabled',
'ExpirationInDays': 180, # ~6 Months
}]
}
encryption = {
'ServerSideEncryptionConfiguration': [{
'ServerSideEncryptionByDefault': {
'SSEAlgorithm': 'AES256'
}
}]
}
config.add_s3_bucket("BackupBucket",
names.backup.s3,
life_cycle_config=life_cycle,
encryption=encryption)
BUCKET_DEPENDENCY = "BackupBucket"
# Vault Backup
cmd = "/usr/local/bin/python3 ~/vault.py backup {}".format(
bosslet_config.INTERNAL_DOMAIN)
pipeline = DataPipeline(log_uri=s3_logs, resource_role="backup")
pipeline.add_shell_command("VaultBackup",
cmd,
destination=Ref("VaultBucket"),
runs_on=Ref("VaultInstance"))
pipeline.add_s3_bucket("VaultBucket", s3_backup + "/vault")
pipeline.add_ec2_instance("VaultInstance",
subnet=internal_subnet,
image=backup_image)
config.add_data_pipeline("VaultBackupPipeline",
"vault-backup." + bosslet_config.INTERNAL_DOMAIN,
pipeline.objects,
depends_on=BUCKET_DEPENDENCY)
# DynamoDB Backup
tables = {
"BossMeta": names.meta.ddb,
"S3Index": names.s3_index.ddb,
"TileIndex": names.tile_index.ddb,
"IdIndex": names.id_index.ddb,
"IdCountIndex": names.id_count_index.ddb,
"VaultData": names.vault.ddb,
}
pipeline = DataPipeline(log_uri=s3_logs)
pipeline.add_emr_cluster("BackupCluster", region=bosslet_config.REGION)
for name in tables:
table = tables[name]
pipeline.add_s3_bucket(name + "Bucket", s3_backup + "/DDB/" + table)
pipeline.add_ddb_table(name, table)
pipeline.add_emr_copy(name + "Copy",
Ref(name),
Ref(name + "Bucket"),
runs_on=Ref("BackupCluster"),
region=bosslet_config.REGION)
config.add_data_pipeline("DDBPipeline",
"dynamo-backup." + bosslet_config.INTERNAL_DOMAIN,
pipeline.objects,
depends_on=BUCKET_DEPENDENCY)
# Endpoint RDS Backup
pipeline = rds_copy(names.endpoint_db.rds, internal_subnet, backup_image,
s3_logs, s3_backup)
config.add_data_pipeline("EndpointPipeline",
"endpoint-backup." +
bosslet_config.INTERNAL_DOMAIN,
pipeline.objects,
depends_on=BUCKET_DEPENDENCY)
# Auth RDS Backup
if bosslet_config.AUTH_RDS:
pipeline = rds_copy(names.auth_db.rds, internal_subnet, backup_image,
s3_logs, s3_backup)
config.add_data_pipeline("AuthPipeline",
"auth-backup." +
bosslet_config.INTERNAL_DOMAIN,
pipeline.objects,
depends_on=BUCKET_DEPENDENCY)
return config
def create_config(bosslet_config, user_data=None):
"""
Create the CloudFormationConfiguration object.
Args:
bosslet_config (BossConfiguration): target bosslet
user_data (UserData): information used by the endpoint instance and vault. Data will be run through the CloudFormation Fn::Join template intrinsic function so other template intrinsic functions used in the user_data will be parsed and executed.
Returns: the config for the Cloud Formation stack
"""
# Prepare user data for parsing by CloudFormation.
if user_data is not None:
parsed_user_data = { "Fn::Join" : ["", user_data.format_for_cloudformation()]}
else:
parsed_user_data = user_data
keypair = bosslet_config.SSH_KEY
session = bosslet_config.session
names = bosslet_config.names
config = CloudFormationConfiguration("cachedb", bosslet_config)
vpc_id = config.find_vpc()
#####
# TODO: When CF config files are refactored for multi-account support
# the creation of _all_ subnets should be moved into core.
# AWS doesn't charge for the VPC or subnets, so it doesn't
# increase cost and cleans up subnet creation
# Create several subnets for all the lambdas to use.
internal_route_table_id = aws.rt_lookup(session, vpc_id, names.internal.rt)
lambda_subnets = config.add_all_lambda_subnets()
for lambda_subnet in lambda_subnets:
key = lambda_subnet['Ref']
config.add_route_table_association(key + "RTA",
internal_route_table_id,
lambda_subnet)
# Create a custom resource to help delete ENIs from lambdas
# DP NOTE: After deleting a lambda the ENIs may stick around for while, causing the stack delete to fail
# See https://stackoverflow.com/a/41310289
config.add_arg(Arg.String('StackName', config.stack_name))
config.add_arg(Arg.String('DeleteENILambda', aws.lambda_arn_lookup(session, names.delete_eni.lambda_)))
config.add_custom_resource('DeleteENI', 'DeleteENI', Ref('DeleteENILambda'), StackName = Ref('StackName'))
# Lookup the External Subnet, Internal Security Group IDs that are
# needed by other resources
internal_subnet_id = aws.subnet_id_lookup(session, names.internal.subnet)
config.add_arg(Arg.Subnet("InternalSubnet",
internal_subnet_id,
"ID of Internal Subnet to create resources in"))
internal_sg_id = aws.sg_lookup(session, vpc_id, names.internal.sg)
config.add_arg(Arg.SecurityGroup("InternalSecurityGroup",
internal_sg_id,
"ID of internal Security Group"))
role = aws.role_arn_lookup(session, "lambda_cache_execution")
config.add_arg(Arg.String("LambdaCacheExecutionRole", role,
"IAM role for " + names.multi_lambda.lambda_))
cuboid_import_role = aws.role_arn_lookup(session, CUBOID_IMPORT_ROLE)
config.add_arg(Arg.String(CUBOID_IMPORT_ROLE, cuboid_import_role,
"IAM role for cuboidImport"))
config.add_capabilities(['CAPABILITY_IAM'])
cuboid_bucket_name = names.cuboid_bucket.s3
if not aws.s3_bucket_exists(session, cuboid_bucket_name):
config.add_s3_bucket("cuboidBucket", cuboid_bucket_name)
config.add_s3_bucket_policy(
"cuboidBucketPolicy", cuboid_bucket_name,
['s3:GetObject', 's3:PutObject'],
{ 'AWS': role})
config.append_s3_bucket_policy(
"cuboidBucketPolicy", cuboid_bucket_name,
['s3:PutObject'], { 'AWS': cuboid_import_role})
delete_bucket_name = names.delete_bucket.s3
if not aws.s3_bucket_exists(session, delete_bucket_name):
config.add_s3_bucket("deleteBucket", delete_bucket_name)
config.add_s3_bucket_policy(
"deleteBucketPolicy", delete_bucket_name,
['s3:GetObject', 's3:PutObject'],
{ 'AWS': role})
tile_bucket_name = names.tile_bucket.s3
if not aws.s3_bucket_exists(session, tile_bucket_name):
life_cycle_cfg = get_cf_bucket_life_cycle_rules()
config.add_s3_bucket(
"tileBucket", tile_bucket_name, life_cycle_config=life_cycle_cfg)
config.add_s3_bucket_policy(
"tileBucketPolicy", tile_bucket_name,
['s3:GetObject', 's3:PutObject'],
{ 'AWS': role})
# The ingest bucket is a staging area for cuboids uploaded during volumetric ingest.
creating_ingest_bucket = False
ingest_bucket_name = names.ingest_bucket.s3
if not aws.s3_bucket_exists(session, ingest_bucket_name):
creating_ingest_bucket = True
ing_bucket_life_cycle_cfg = get_cf_bucket_life_cycle_rules()
config.add_s3_bucket("ingestBucket", ingest_bucket_name,
life_cycle_config=ing_bucket_life_cycle_cfg)
config.add_s3_bucket_policy(
"ingestBucketPolicy", ingest_bucket_name,
['s3:GetObject', 's3:PutObject', 's3:PutObjectTagging'],
{ 'AWS': cuboid_import_role})
config.add_ec2_instance("CacheManager",
names.cachemanager.dns,
aws.ami_lookup(bosslet_config, names.cachemanager.ami),
keypair,
subnet=Ref("InternalSubnet"),
public_ip=False,
type_=const.CACHE_MANAGER_TYPE,
security_groups=[Ref("InternalSecurityGroup")],
user_data=parsed_user_data,
role="cachemanager")
config.add_sqs_queue(
names.ingest_cleanup_dlq.sqs, names.ingest_cleanup_dlq.sqs, 30, 20160)
config.add_sqs_queue(
names.cuboid_import_dlq.sqs, names.cuboid_import_dlq.sqs, 30, 20160)
config.add_sqs_policy('cuboidImportDlqPolicy', 'cuboidImportDlqPolicy',
[Ref(names.cuboid_import_dlq.sqs)], cuboid_import_role)
config.add_lambda("MultiLambda",
names.multi_lambda.lambda_,
Ref("LambdaCacheExecutionRole"),
handler='lambda_loader.handler',
timeout=120,
memory=1536,
security_groups=[Ref('InternalSecurityGroup')],
subnets=lambda_subnets)
config.add_lambda("TileUploadedLambda",
names.tile_uploaded.lambda_,
Ref("LambdaCacheExecutionRole"),
handler='tile_uploaded_lambda.handler',
timeout=5,
memory=1024)
config.add_lambda("TileIngestLambda",
names.tile_ingest.lambda_,
Ref("LambdaCacheExecutionRole"),
handler='tile_ingest_lambda.handler',
timeout=30,
memory=1536)
config.add_lambda("DeleteTileObjsLambda",
names.delete_tile_objs.lambda_,
Ref("LambdaCacheExecutionRole"),
handler='delete_tile_objs_lambda.handler',
timeout=90,
memory=128,
dlq=Arn(names.ingest_cleanup_dlq.sqs))
config.add_lambda("DeleteTileEntryLambda",
names.delete_tile_index_entry.lambda_,
Ref("LambdaCacheExecutionRole"),
handler='delete_tile_index_entry_lambda.handler',
timeout=90,
memory=128,
dlq=Arn(names.ingest_cleanup_dlq.sqs))
config.add_lambda("CuboidImportLambda",
names.cuboid_import_lambda.lambda_,
Ref(CUBOID_IMPORT_ROLE),
handler='cuboid_import_lambda.handler',
timeout=90,
memory=128,
dlq=Arn(names.cuboid_import_dlq.sqs))
config.add_lambda("VolumetricIngestLambda",
names.volumetric_ingest_queue_upload_lambda.lambda_,
Ref("LambdaCacheExecutionRole"),
handler='ingest_queue_upload_volumetric_lambda.handler',
timeout=120,
memory=1024)
if creating_ingest_bucket:
config.add_lambda_permission(
'ingestBucketInvokeCuboidImportLambda', names.cuboid_import_lambda.lambda_,
principal='s3.amazonaws.com', source={
'Fn::Join': [':', ['arn', 'aws', 's3', '', '', ingest_bucket_name]]}, #DP TODO: move into constants
depends_on=['ingestBucket', 'CuboidImportLambda']
)
else:
# NOTE: this permission doesn't seem to apply properly when doing a
# CloudFormation update. During testing, I had to manually apply this
# permission before the bucket trigger could be applied in post_init().
# Doing a CloudFormation delete followed by a create did not have a
# problem.
config.add_lambda_permission(
'ingestBucketInvokeCuboidImportLambda', names.cuboid_import_lambda.lambda_,
principal='s3.amazonaws.com', source={
'Fn::Join': [':', ['arn', 'aws', 's3', '', '', ingest_bucket_name]]},
depends_on='CuboidImportLambda'
)
# Add topic to indicating that the object store has been write locked.
# Now using "production mailing list" instead of separate write lock topic.
#config.add_sns_topic('WriteLock',
# names.write_lock_topic,
# names.write_lock,
# []) # TODO: add subscribers
return config
def create_config(session, domain, keypair=None, user_data=None):
"""
Create the CloudFormationConfiguration object.
Args:
session: amazon session object
domain: domain of the stack being created
keypair: keypair used to by instances being created
user_data (UserData): information used by the endpoint instance and vault. Data will be run through the CloudFormation Fn::Join template intrinsic function so other template intrinsic functions used in the user_data will be parsed and executed.
Returns: the config for the Cloud Formation stack
"""
# Prepare user data for parsing by CloudFormation.
if user_data is not None:
parsed_user_data = { "Fn::Join" : ["", user_data.format_for_cloudformation()]}
else:
parsed_user_data = user_data
names = AWSNames(domain)
config = CloudFormationConfiguration("cachedb", domain, const.REGION)
vpc_id = config.find_vpc(session)
# Create several subnets for all the lambdas to use.
lambda_azs = aws.azs_lookup(session, lambda_compatible_only=True)
internal_route_table_id = aws.rt_lookup(session, vpc_id, names.internal)
print("AZs for lambda: " + str(lambda_azs))
lambda_subnets = []
for i in range(const.LAMBDA_SUBNETS):
key = 'LambdaSubnet{}'.format(i)
lambda_subnets.append(Ref(key))
config.add_subnet(key, names.subnet('lambda{}'.format(i)), az=lambda_azs[i % len(lambda_azs)][0])
config.add_route_table_association(key + "RTA",
internal_route_table_id,
Ref(key))
# Lookup the External Subnet, Internal Security Group IDs that are
# needed by other resources
internal_subnet_id = aws.subnet_id_lookup(session, names.subnet("internal"))
config.add_arg(Arg.Subnet("InternalSubnet",
internal_subnet_id,
"ID of Internal Subnet to create resources in"))
internal_sg_id = aws.sg_lookup(session, vpc_id, names.internal)
config.add_arg(Arg.SecurityGroup("InternalSecurityGroup",
internal_sg_id,
"ID of internal Security Group"))
role = aws.role_arn_lookup(session, "lambda_cache_execution")
config.add_arg(Arg.String("LambdaCacheExecutionRole", role,
"IAM role for multilambda." + domain))
cuboid_import_role = aws.role_arn_lookup(session, CUBOID_IMPORT_ROLE)
config.add_arg(Arg.String(CUBOID_IMPORT_ROLE, cuboid_import_role,
"IAM role for cuboidImport." + domain))
config.add_capabilities(['CAPABILITY_IAM'])
# Allow updating S3 index table with cuboid's object key during
# volumetric ingest.
# Example of s3_index_arn form: arn:aws:dynamodb:us-east-1:12345678:table/s3index.*.boss
config.add_iam_policy_to_role(
'S3IndexPutItem{}'.format(domain).replace('.', ''),
get_s3_index_arn(session, domain).replace(domain,'*.') + domain.split('.')[1],
[CUBOID_IMPORT_ROLE], ['dynamodb:PutItem'])
cuboid_bucket_name = names.cuboid_bucket
if not aws.s3_bucket_exists(session, cuboid_bucket_name):
config.add_s3_bucket("cuboidBucket", cuboid_bucket_name)
config.add_s3_bucket_policy(
"cuboidBucketPolicy", cuboid_bucket_name,
['s3:GetObject', 's3:PutObject'],
{ 'AWS': role})
config.append_s3_bucket_policy(
"cuboidBucketPolicy", cuboid_bucket_name,
['s3:PutObject'], { 'AWS': cuboid_import_role})
delete_bucket_name = names.delete_bucket
if not aws.s3_bucket_exists(session, delete_bucket_name):
config.add_s3_bucket("deleteBucket", delete_bucket_name)
config.add_s3_bucket_policy(
"deleteBucketPolicy", delete_bucket_name,
['s3:GetObject', 's3:PutObject'],
{ 'AWS': role})
tile_bucket_name = names.tile_bucket
if not aws.s3_bucket_exists(session, tile_bucket_name):
life_cycle_cfg = get_cf_bucket_life_cycle_rules()
config.add_s3_bucket(
"tileBucket", tile_bucket_name, life_cycle_config=life_cycle_cfg)
config.add_s3_bucket_policy(
"tileBucketPolicy", tile_bucket_name,
['s3:GetObject', 's3:PutObject'],
{ 'AWS': role})
# The ingest bucket is a staging area for cuboids uploaded during volumetric ingest.
creating_ingest_bucket = False
ingest_bucket_name = names.ingest_bucket
if not aws.s3_bucket_exists(session, ingest_bucket_name):
creating_ingest_bucket = True
ing_bucket_life_cycle_cfg = get_cf_bucket_life_cycle_rules()
config.add_s3_bucket("ingestBucket", ingest_bucket_name,
life_cycle_config=ing_bucket_life_cycle_cfg)
config.add_s3_bucket_policy(
"ingestBucketPolicy", ingest_bucket_name,
['s3:GetObject', 's3:PutObject', 's3:PutObjectTagging'],
{ 'AWS': cuboid_import_role})
config.add_ec2_instance("CacheManager",
names.cache_manager,
aws.ami_lookup(session, "cachemanager.boss"),
keypair,
subnet=Ref("InternalSubnet"),
public_ip=False,
type_=const.CACHE_MANAGER_TYPE,
security_groups=[Ref("InternalSecurityGroup")],
user_data=parsed_user_data,
role="cachemanager")
config.add_sqs_queue(
names.ingest_cleanup_dlq, names.ingest_cleanup_dlq, 30, 20160)
config.add_sqs_queue(
names.cuboid_import_dlq, names.cuboid_import_dlq, 30, 20160)
config.add_sqs_policy('cuboidImportDlqPolicy', 'cuboidImportDlqPolicy',
[Ref(names.cuboid_import_dlq)], cuboid_import_role)
lambda_bucket = aws.get_lambda_s3_bucket(session)
config.add_lambda("MultiLambda",
names.multi_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"lambda_loader.handler"),
timeout=120,
memory=1536,
security_groups=[Ref('InternalSecurityGroup')],
subnets=lambda_subnets,
runtime='python3.6')
config.add_lambda("TileUploadedLambda",
names.tile_uploaded_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"tile_uploaded_lambda.handler"),
timeout=5,
memory=1024,
runtime='python3.6')
config.add_lambda("TileIngestLambda",
names.tile_ingest_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"tile_ingest_lambda.handler"),
timeout=30,
memory=1536,
runtime='python3.6')
config.add_lambda("DeleteTileObjsLambda",
names.delete_tile_objs_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"delete_tile_objs_lambda.handler"),
timeout=90,
memory=128,
runtime='python3.6',
dlq=Arn(names.ingest_cleanup_dlq))
config.add_lambda("DeleteTileEntryLambda",
names.delete_tile_index_entry_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"delete_tile_index_entry_lambda.handler"),
timeout=90,
memory=128,
runtime='python3.6',
dlq=Arn(names.ingest_cleanup_dlq))
config.add_lambda("CuboidImportLambda",
names.cuboid_import_lambda,
Ref(CUBOID_IMPORT_ROLE),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"cuboid_import_lambda.handler"),
timeout=90,
memory=128,
runtime='python3.6',
dlq=Arn(names.cuboid_import_dlq))
config.add_lambda("VolumetricIngestLambda",
names.volumetric_ingest_queue_upload_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(lambda_bucket,
"multilambda.{}.zip".format(domain),
"ingest_queue_upload_volumetric_lambda.handler"),
timeout=120,
memory=1024,
runtime='python3.6')
if creating_ingest_bucket:
config.add_lambda_permission(
'ingestBucketInvokeCuboidImportLambda', names.cuboid_import_lambda,
principal='s3.amazonaws.com', source={
'Fn::Join': [':', ['arn', 'aws', 's3', '', '', ingest_bucket_name]]}, #DP TODO: move into constants
depends_on=['ingestBucket', 'CuboidImportLambda']
)
else:
# NOTE: this permission doesn't seem to apply properly when doing a
# CloudFormation update. During testing, I had to manually apply this
# permission before the bucket trigger could be applied in post_init().
# Doing a CloudFormation delete followed by a create did not have a
# problem.
config.add_lambda_permission(
'ingestBucketInvokeCuboidImportLambda', names.cuboid_import_lambda,
principal='s3.amazonaws.com', source={
'Fn::Join': [':', ['arn', 'aws', 's3', '', '', ingest_bucket_name]]},
depends_on='CuboidImportLambda'
)
# Add topic to indicating that the object store has been write locked.
# Now using "production mailing list" instead of separate write lock topic.
#config.add_sns_topic('WriteLock',
# names.write_lock_topic,
# names.write_lock,
# []) # TODO: add subscribers
return config
def create_config(session, domain, keypair=None, user_data=None):
"""
Create the CloudFormationConfiguration object.
Args:
session: amazon session object
domain: domain of the stack being created
keypair: keypair used to by instances being created
user_data (UserData): information used by the endpoint instance and vault. Data will be run through the CloudFormation Fn::Join template intrinsic function so other template intrinsic functions used in the user_data will be parsed and executed.
Returns: the config for the Cloud Formation stack
"""
# Prepare user data for parsing by CloudFormation.
if user_data is not None:
parsed_user_data = {
"Fn::Join": ["", user_data.format_for_cloudformation()]
}
else:
parsed_user_data = user_data
names = AWSNames(domain)
config = CloudFormationConfiguration("cachedb", domain, const.REGION)
vpc_id = config.find_vpc(session)
# Create several subnets for all the lambdas to use.
lambda_azs = aws.azs_lookup(session, lambda_compatible_only=True)
internal_route_table_id = aws.rt_lookup(session, vpc_id, names.internal)
print("AZs for lambda: " + str(lambda_azs))
lambda_subnets = []
for i in range(const.LAMBDA_SUBNETS):
key = 'LambdaSubnet{}'.format(i)
lambda_subnets.append(Ref(key))
config.add_subnet(key,
names.subnet('lambda{}'.format(i)),
az=lambda_azs[i % len(lambda_azs)][0])
config.add_route_table_association(key + "RTA",
internal_route_table_id, Ref(key))
# Lookup the External Subnet, Internal Security Group IDs that are
# needed by other resources
internal_subnet_id = aws.subnet_id_lookup(session,
names.subnet("internal"))
config.add_arg(
Arg.Subnet("InternalSubnet", internal_subnet_id,
"ID of Internal Subnet to create resources in"))
internal_sg_id = aws.sg_lookup(session, vpc_id, names.internal)
config.add_arg(
Arg.SecurityGroup("InternalSecurityGroup", internal_sg_id,
"ID of internal Security Group"))
role = aws.role_arn_lookup(session, "lambda_cache_execution")
config.add_arg(
Arg.String("LambdaCacheExecutionRole", role,
"IAM role for multilambda." + domain))
index_bucket_name = names.cuboid_bucket
if not aws.s3_bucket_exists(session, index_bucket_name):
config.add_s3_bucket("cuboidBucket", index_bucket_name)
config.add_s3_bucket_policy("cuboidBucketPolicy", index_bucket_name,
['s3:GetObject', 's3:PutObject'],
{'AWS': role})
delete_bucket_name = names.delete_bucket
if not aws.s3_bucket_exists(session, delete_bucket_name):
config.add_s3_bucket("deleteBucket", delete_bucket_name)
config.add_s3_bucket_policy("deleteBucketPolicy", delete_bucket_name,
['s3:GetObject', 's3:PutObject'],
{'AWS': role})
creating_tile_bucket = False
tile_bucket_name = names.tile_bucket
if not aws.s3_bucket_exists(session, tile_bucket_name):
creating_tile_bucket = True
config.add_s3_bucket("tileBucket", tile_bucket_name)
config.add_s3_bucket_policy("tileBucketPolicy", tile_bucket_name,
['s3:GetObject', 's3:PutObject'],
{'AWS': role})
ingest_bucket_name = names.ingest_bucket
if not aws.s3_bucket_exists(session, ingest_bucket_name):
config.add_s3_bucket("ingestBucket", ingest_bucket_name)
config.add_s3_bucket_policy("ingestBucketPolicy", ingest_bucket_name,
['s3:GetObject', 's3:PutObject'],
{'AWS': role})
config.add_ec2_instance("CacheManager",
names.cache_manager,
aws.ami_lookup(session, "cachemanager.boss"),
keypair,
subnet=Ref("InternalSubnet"),
public_ip=False,
type_=const.CACHE_MANAGER_TYPE,
security_groups=[Ref("InternalSecurityGroup")],
user_data=parsed_user_data,
role="cachemanager")
lambda_bucket = aws.get_lambda_s3_bucket(session)
config.add_lambda("MultiLambda",
names.multi_lambda,
Ref("LambdaCacheExecutionRole"),
s3=(aws.get_lambda_s3_bucket(session),
"multilambda.{}.zip".format(domain),
"lambda_loader.handler"),
timeout=120,
memory=1024,
security_groups=[Ref('InternalSecurityGroup')],
subnets=lambda_subnets,
runtime='python3.6')
if creating_tile_bucket:
config.add_lambda_permission(
'tileBucketInvokeMultiLambda',
names.multi_lambda,
principal='s3.amazonaws.com',
source={
'Fn::Join':
[':', ['arn', 'aws', 's3', '', '', tile_bucket_name]]
}, #DP TODO: move into constants
depends_on=['tileBucket', 'MultiLambda'])
else:
config.add_lambda_permission(
'tileBucketInvokeMultiLambda',
names.multi_lambda,
principal='s3.amazonaws.com',
source={
'Fn::Join':
[':', ['arn', 'aws', 's3', '', '', tile_bucket_name]]
},
depends_on='MultiLambda')
# Add topic to indicating that the object store has been write locked.
# Now using "production mailing list" instead of separate write lock topic.
#config.add_sns_topic('WriteLock',
# names.write_lock_topic,
# names.write_lock,
# []) # TODO: add subscribers
return config