def create_config(session, domain, keypair=None):
"""
Create the CloudFormationConfiguration object.
Args:
session: amazon session object
domain (string): domain of the stack being created
keypair: keypair used to by instances being created
Returns: the config for the Cloud Formation stack
"""
names = AWSNames(domain)
config = CloudFormationConfiguration('redis', domain, const.REGION)
vpc_id = config.find_vpc(session)
az_subnets, external_subnets = config.find_all_availability_zones(session)
sgs = aws.sg_lookup_all(session, vpc_id)
# Create the Cache and CacheState Redis Clusters
REDIS_PARAMETERS = {
"maxmemory-policy": "volatile-lru",
"reserved-memory": str(get_scenario(const.REDIS_RESERVED_MEMORY, 0) * 1000000),
"maxmemory-samples": "5", # ~ 5 - 10
}
config.add_redis_replication("Cache",
names.cache,
az_subnets,
[sgs[names.internal]],
type_=const.REDIS_CACHE_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE,
parameters=REDIS_PARAMETERS)
config.add_redis_replication("CacheState",
names.cache_state,
az_subnets,
[sgs[names.internal]],
type_=const.REDIS_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE)
# This one may not be created depending on the scenario type.
if get_scenario(const.REDIS_SESSION_TYPE, None) is not None:
config.add_redis_replication("CacheSession",
names.cache_session,
az_subnets,
[sgs[names.internal]],
type_=const.REDIS_SESSION_TYPE,
version="3.2.4",
clusters=1)
return config
def create_config(session, domain, keypair=None):
"""
Create the CloudFormationConfiguration object.
Args:
session: amazon session object
domain (string): domain of the stack being created
keypair: keypair used to by instances being created
Returns: the config for the Cloud Formation stack
"""
names = AWSNames(domain)
config = CloudFormationConfiguration('redis', domain, const.REGION)
vpc_id = config.find_vpc(session)
az_subnets, external_subnets = config.find_all_availability_zones(session)
sgs = aws.sg_lookup_all(session, vpc_id)
# Create the Cache and CacheState Redis Clusters
REDIS_PARAMETERS = {
"maxmemory-policy": "volatile-lru",
"reserved-memory": str(get_scenario(const.REDIS_RESERVED_MEMORY, 0) * 1000000),
"maxmemory-samples": "5", # ~ 5 - 10
}
config.add_redis_replication("Cache",
names.cache,
az_subnets,
[sgs[names.internal]],
type_=const.REDIS_CACHE_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE,
parameters=REDIS_PARAMETERS)
config.add_redis_replication("CacheState",
names.cache_state,
az_subnets,
[sgs[names.internal]],
type_=const.REDIS_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE)
return config
def create_config(bosslet_config):
names = bosslet_config.names
session = bosslet_config.session
config = CloudFormationConfiguration('redis', bosslet_config)
vpc_id = config.find_vpc()
internal_subnets, external_subnets = config.find_all_subnets()
sgs = aws.sg_lookup_all(session, vpc_id)
# Create the Cache and CacheState Redis Clusters
REDIS_PARAMETERS = {
"maxmemory-policy": "volatile-lru",
"reserved-memory-percent": str(const.REDIS_RESERVED_MEMORY_PERCENT),
"maxmemory-samples": "5", # ~ 5 - 10
}
config.add_redis_replication("Cache",
names.cache.redis,
internal_subnets,
[sgs[names.internal.sg]],
type_=const.REDIS_CACHE_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE,
parameters=REDIS_PARAMETERS)
config.add_redis_replication("CacheState",
names.cache_state.redis,
internal_subnets,
[sgs[names.internal.sg]],
type_=const.REDIS_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE)
# This one may not be created depending on the scenario type.
if const.REDIS_SESSION_TYPE is not None:
config.add_redis_replication("CacheSession",
names.cache_session.redis,
internal_subnets,
[sgs[names.internal.sg]],
type_=const.REDIS_SESSION_TYPE,
version="3.2.4",
clusters=1)
return config
def create_config(session, domain, keypair=None, user_data=None, db_config={}):
"""Create the CloudFormationConfiguration object."""
names = AWSNames(domain)
config = CloudFormationConfiguration('proofreader', domain, const.REGION)
vpc_id = config.find_vpc(session)
az_subnets, _ = config.find_all_availability_zones(session)
external_subnet_id = aws.subnet_id_lookup(session,
names.subnet("external"))
config.add_arg(
Arg.Subnet("ExternalSubnet", external_subnet_id,
"ID of External Subnet to create resources in"))
sgs = aws.sg_lookup_all(session, vpc_id)
# Only allow unsecure web access from APL, until a ELB is configured for HTTPS
config.add_security_group("HttpSecurityGroup", names.http,
[("tcp", "80", "80", const.INCOMING_SUBNET)])
config.add_ec2_instance("ProofreaderWeb",
names.proofreader,
aws.ami_lookup(session, "proofreader-web.boss"),
keypair,
public_ip=True,
subnet=Ref("ExternalSubnet"),
security_groups=[
sgs[names.internal], sgs[names.ssh],
Ref('HttpSecurityGroup')
],
user_data=user_data,
depends_on="ProofreaderDB"
) # make sure the DB is launched before we start
config.add_rds_db("ProofreaderDB",
names.proofreader_db,
db_config.get("port"),
db_config.get("name"),
db_config.get("user"),
db_config.get("password"),
az_subnets,
security_groups=[sgs[names.internal]])
return config
def create_config(bosslet_config, lookup=True):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('activities', bosslet_config)
names = bosslet_config.names
keypair = bosslet_config.SSH_KEY
session = bosslet_config.session
vpc_id = config.find_vpc()
sgs = aws.sg_lookup_all(session, vpc_id)
internal_subnets, _ = config.find_all_subnets()
internal_subnets_asg, _ = config.find_all_subnets(compatibility='asg')
topic_arn = aws.sns_topic_lookup(session, bosslet_config.ALERT_TOPIC)
if topic_arn is None:
raise MissingResourceError('SNS topic', bosslet_config.ALERT_TOPIC)
event_data = {
"lambda-name": "delete_lambda",
"db": names.endpoint_db.rds,
"meta-db": names.meta.ddb,
"s3-index-table": names.s3_index.ddb,
"id-index-table": names.id_index.ddb,
"id-count-table": names.id_count_index.ddb,
"cuboid_bucket": names.cuboid_bucket.s3,
"delete_bucket": names.delete_bucket.s3,
"topic-arn": topic_arn,
"query-deletes-sfn-name": names.query_deletes.sfn,
"delete-sfn-name": names.delete_cuboid.sfn,
"delete-exp-sfn-name": names.delete_experiment.sfn,
"delete-coord-frame-sfn-name": names.delete_coord_frame.sfn,
"delete-coll-sfn-name": names.delete_collection.sfn,
}
role_arn = aws.role_arn_lookup(session, "events_for_delete_lambda")
multi_lambda = names.multi_lambda.lambda_
if lookup:
lambda_arn = aws.lambda_arn_lookup(session, multi_lambda)
else:
lambda_arn = None
target_list = [{
"Arn": lambda_arn,
"Id": multi_lambda,
"Input": json.dumps(event_data)
}]
schedule_expression = "cron(1 6-11/1 ? * TUE-FRI *)"
#schedule_expression = "cron(0/2 * * * ? *)" # testing fire every two minutes
config.add_event_rule("DeleteEventRule",
# XXX What type for event rules?
names.delete_event_rule.dns,
role_arn=role_arn,
schedule_expression=schedule_expression,
target_list=target_list,
state='DISABLED') # Disabled until new delete is finished.
# Events have to be given permission to run lambda.
config.add_lambda_permission('DeleteRulePerm',
multi_lambda,
principal='events.amazonaws.com',
source=Arn('DeleteEventRule'))
user_data = UserData()
user_data["system"]["fqdn"] = names.activities.dns
user_data["system"]["type"] = "activities"
user_data["aws"]["db"] = names.endpoint_db.rds
user_data["aws"]["cache"] = names.cache.redis
user_data["aws"]["cache-state"] = names.cache_state.redis
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["meta-db"] = names.meta.ddb
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket.s3
user_data["aws"]["tile_bucket"] = names.tile_bucket.s3
user_data["aws"]["ingest_bucket"] = names.ingest_bucket.s3
user_data["aws"]["s3-index-table"] = names.s3_index.ddb
user_data["aws"]["tile-index-table"] = names.tile_index.ddb
user_data["aws"]["id-index-table"] = names.id_index.ddb
user_data["aws"]["id-count-table"] = names.id_count_index.ddb
user_data["aws"]["max_task_id_suffix"] = str(const.MAX_TASK_ID_SUFFIX)
user_data["aws"]["tile_ingest_lambda"] = names.tile_ingest.lambda_
user_data["aws"]["tile_uploaded_lambda"] = names.tile_uploaded.lambda_
config.add_autoscale_group("Activities",
names.activities.dns,
aws.ami_lookup(bosslet_config, names.activities.ami),
keypair,
subnets=internal_subnets_asg,
type_=const.ACTIVITIES_TYPE,
security_groups=[sgs[names.internal.sg]],
user_data=str(user_data),
role=aws.instance_profile_arn_lookup(session, "activities"),
min=1,
max=1)
config.add_lambda("IngestLambda",
names.ingest_lambda.lambda_,
aws.role_arn_lookup(session, 'IngestQueueUpload'),
const.INGEST_LAMBDA,
handler="index.handler",
timeout=60 * 5,
runtime='python3.6',
memory=3008)
config.add_lambda_permission("IngestLambdaExecute", Ref("IngestLambda"))
# Downsample / Resolution Hierarchy support
lambda_role = aws.role_arn_lookup(session, "lambda_resolution_hierarchy")
config.add_lambda("DownsampleVolumeLambda",
names.downsample_volume.lambda_,
lambda_role,
handler="downsample_volume.handler",
timeout=120,
memory=1024,
dlq = Ref('DownsampleDLQ'))
start_sfn_lambda_role = aws.role_arn_lookup(session, 'StartStepFcnLambdaRole')
config.add_lambda("startSfnLambda",
names.start_sfn.lambda_,
start_sfn_lambda_role,
handler="start_sfn_lambda.handler",
timeout=60,
memory=128)
# This dead letter queue behavior uses a lambda to put failed lambda
# executions into a dlqs created specifically for each downsample job.
# There is a separate dlq for each resolution.
config.add_sns_topic("DownsampleDLQ",
names.downsample_dlq.sns,
names.downsample_dlq.sns,
[('lambda', Arn('DownsampleDLQLambda'))])
config.add_lambda('DownsampleDLQLambda',
names.downsample_dlq.lambda_,
lambda_role,
const.DOWNSAMPLE_DLQ_LAMBDA,
handler='index.handler',
runtime='python3.7',
timeout=10)
config.add_lambda_permission('DownsampleDLQLambdaExecute',
Ref('DownsampleDLQLambda'))
return config
def create_config(session, domain):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('activities', domain, const.REGION)
names = AWSNames(domain)
global keypair
keypair = aws.keypair_lookup(session)
vpc_id = config.find_vpc(session)
sgs = aws.sg_lookup_all(session, vpc_id)
internal_subnets, _ = config.find_all_availability_zones(session)
internal_subnets_lambda, _ = config.find_all_availability_zones(
session, lambda_compatible_only=True)
topic_arn = aws.sns_topic_lookup(session, "ProductionMicronsMailingList")
event_data = {
"lambda-name": "delete_lambda",
"db": names.endpoint_db,
"meta-db": names.meta,
"s3-index-table": names.s3_index,
"id-index-table": names.id_index,
"id-count-table": names.id_count_index,
"cuboid_bucket": names.cuboid_bucket,
"delete_bucket": names.delete_bucket,
"topic-arn": topic_arn,
"query-deletes-sfn-name": names.query_deletes,
"delete-sfn-name": names.delete_cuboid,
"delete-exp-sfn-name": names.delete_experiment,
"delete-coord-frame-sfn-name": names.delete_coord_frame,
"delete-coll-sfn-name": names.delete_collection
}
role_arn = aws.role_arn_lookup(session, "events_for_delete_lambda")
multi_lambda = names.multi_lambda
lambda_arn = aws.lambda_arn_lookup(session, multi_lambda)
target_list = [{
"Arn": lambda_arn,
"Id": multi_lambda,
"Input": json.dumps(event_data)
}]
schedule_expression = "cron(1 6-11/1 ? * TUE-FRI *)"
#schedule_expression = "cron(0/2 * * * ? *)" # testing fire every two minutes
config.add_event_rule("DeleteEventRule",
names.delete_event_rule,
role_arn=role_arn,
schedule_expression=schedule_expression,
target_list=target_list,
description=None)
# Events have to be given permission to run lambda.
config.add_lambda_permission('DeleteRulePerm',
multi_lambda,
principal='events.amazonaws.com',
source=Arn('DeleteEventRule'))
user_data = UserData()
user_data["system"]["fqdn"] = names.activities
user_data["system"]["type"] = "activities"
user_data["aws"]["db"] = names.endpoint_db
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["meta-db"] = names.meta
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["tile_bucket"] = names.tile_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["tile-index-table"] = names.tile_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
config.add_autoscale_group("Activities",
names.activities,
aws.ami_lookup(session, 'activities.boss'),
keypair,
subnets=internal_subnets_lambda,
type_=const.ACTIVITIES_TYPE,
security_groups=[sgs[names.internal]],
user_data=str(user_data),
role=aws.instance_profile_arn_lookup(
session, "activities"),
min=1,
max=1)
config.add_lambda("IngestLambda",
names.ingest_lambda,
aws.role_arn_lookup(session, 'IngestQueueUpload'),
const.INGEST_LAMBDA,
handler="index.handler",
timeout=60 * 5)
config.add_lambda_permission("IngestLambdaExecute", Ref("IngestLambda"))
return config
def create_config(session, domain, keypair=None, db_config={}):
"""
Create the CloudFormationConfiguration object.
Args:
session: amazon session object
domain (string): domain of the stack being created
keypair: keypair used to by instances being created
db_config (dict): information needed by rds
Returns: the config for the Cloud Formation stack
"""
names = AWSNames(domain)
# Lookup IAM Role and SNS Topic ARNs for used later in the config
endpoint_role_arn = aws.role_arn_lookup(session, "endpoint")
cachemanager_role_arn = aws.role_arn_lookup(session, 'cachemanager')
dns_arn = aws.sns_topic_lookup(session, names.dns.replace(".", "-"))
if dns_arn is None:
raise Exception("SNS topic named dns." + domain + " does not exist.")
mailing_list_arn = aws.sns_topic_lookup(session, const.PRODUCTION_MAILING_LIST)
if mailing_list_arn is None:
msg = "MailingList {} needs to be created before running config".format(const.PRODUCTION_MAILING_LIST)
raise Exception(msg)
# Configure Vault and create the user data config that the endpoint will
# use for connecting to Vault and the DB instance
user_data = UserData()
user_data["system"]["fqdn"] = names.endpoint
user_data["system"]["type"] = "endpoint"
user_data["aws"]["db"] = names.endpoint_db
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
## cache-db and cache-stat-db need to be in user_data for lambda to access them.
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["meta-db"] = names.meta
# Use CloudFormation's Ref function so that queues' URLs are placed into
# the Boss config file.
user_data["aws"]["s3-flush-queue"] = str(Ref(names.s3flush_queue)) # str(Ref("S3FlushQueue")) DP XXX
user_data["aws"]["s3-flush-deadletter-queue"] = str(Ref(names.deadletter_queue)) #str(Ref("DeadLetterQueue")) DP XXX
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["tile_bucket"] = names.tile_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["tile-index-table"] = names.tile_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
user_data["aws"]["prod_mailing_list"] = mailing_list_arn
user_data["auth"]["OIDC_VERIFY_SSL"] = 'True'
user_data["lambda"]["flush_function"] = names.multi_lambda
user_data["lambda"]["page_in_function"] = names.multi_lambda
user_data["lambda"]["ingest_function"] = names.multi_lambda
user_data['sfn']['populate_upload_queue'] = names.ingest_queue_populate
user_data['sfn']['upload_sfn'] = names.ingest_queue_upload
user_data['sfn']['downsample_sfn'] = names.resolution_hierarchy
# Prepare user data for parsing by CloudFormation.
parsed_user_data = { "Fn::Join" : ["", user_data.format_for_cloudformation()]}
config = CloudFormationConfiguration('api', domain, const.REGION)
vpc_id = config.find_vpc(session)
az_subnets, external_subnets = config.find_all_availability_zones(session)
az_subnets_lambda, external_subnets_lambda = config.find_all_availability_zones(session, lambda_compatible_only=True)
sgs = aws.sg_lookup_all(session, vpc_id)
# DP XXX: hack until we can get productio updated correctly
config.add_security_group('AllHTTPSSecurityGroup', 'https.' + domain, [('tcp', '443', '443', '0.0.0.0/0')])
sgs[names.https] = Ref('AllHTTPSSecurityGroup')
# Create SQS queues and apply access control policies.
#config.add_sqs_queue("DeadLetterQueue", names.deadletter_queue, 30, 20160) DP XXX
config.add_sqs_queue(names.deadletter_queue, names.deadletter_queue, 30, 20160)
max_receives = 3
#config.add_sqs_queue("S3FlushQueue", DP XXX
config.add_sqs_queue(names.s3flush_queue,
names.s3flush_queue,
30,
dead=(Arn(names.deadletter_queue), max_receives))
config.add_sqs_policy("sqsEndpointPolicy", 'sqsEndpointPolicy', # DP XXX
[Ref(names.deadletter_queue), Ref(names.s3flush_queue)],
endpoint_role_arn)
config.add_sqs_policy("sqsCachemgrPolicy", 'sqsCachemgrPolicy', # DP XXX
[Ref(names.deadletter_queue), Ref(names.s3flush_queue)],
cachemanager_role_arn)
# Create the endpoint ASG, ELB, and RDS instance
config.add_autoscale_group("Endpoint",
names.endpoint,
aws.ami_lookup(session, "endpoint.boss"),
keypair,
subnets=az_subnets_lambda,
type_=const.ENDPOINT_TYPE,
security_groups=[sgs[names.internal]],
user_data=parsed_user_data,
min=const.ENDPOINT_CLUSTER_MIN,
max=const.ENDPOINT_CLUSTER_MAX,
elb=Ref("EndpointLoadBalancer"),
notifications=dns_arn,
role=aws.instance_profile_arn_lookup(session, 'endpoint'),
health_check_grace_period=90,
detailed_monitoring=True,
depends_on=["EndpointLoadBalancer", "EndpointDB"])
cert = aws.cert_arn_lookup(session, names.public_dns("api"))
config.add_loadbalancer("EndpointLoadBalancer",
names.endpoint_elb,
[("443", "80", "HTTPS", cert)],
subnets=external_subnets_lambda,
security_groups=[sgs[names.internal], sgs[names.https]],
public=True)
# Endpoint servers are not CPU bound typically, so react quickly to load
config.add_autoscale_policy("EndpointScaleUp",
Ref("Endpoint"),
adjustments=[
(0.0, 10, 1), # 12% - 22% Utilization add 1 instance
(10, None, 2) # Above 22% Utilization add 2 instances
],
alarms=[
("CPUUtilization", "Maximum", "GreaterThanThreshold", "12")
],
period=1)
config.add_autoscale_policy("EndpointScaleDown",
Ref("Endpoint"),
adjustments=[
(None, 0.0, -1), # Under 1.5% Utilization remove 1 instance
],
alarms=[
("CPUUtilization", "Average", "LessThanThreshold", "1.5")
],
period=50)
config.add_rds_db("EndpointDB",
names.endpoint_db,
db_config.get("port"),
db_config.get("name"),
db_config.get("user"),
db_config.get("password"),
az_subnets,
type_ = const.RDS_TYPE,
security_groups=[sgs[names.internal]])
# Create the Meta, s3Index, tileIndex, annotation Dynamo tables
with open(const.DYNAMO_METADATA_SCHEMA, 'r') as fh:
dynamo_cfg = json.load(fh)
config.add_dynamo_table_from_json("EndpointMetaDB", names.meta, **dynamo_cfg)
with open(const.DYNAMO_S3_INDEX_SCHEMA, 'r') as s3fh:
dynamo_s3_cfg = json.load(s3fh)
config.add_dynamo_table_from_json('s3Index', names.s3_index, **dynamo_s3_cfg) # DP XXX
with open(const.DYNAMO_TILE_INDEX_SCHEMA, 'r') as tilefh:
dynamo_tile_cfg = json.load(tilefh)
config.add_dynamo_table_from_json('tileIndex', names.tile_index, **dynamo_tile_cfg) # DP XXX
with open(const.DYNAMO_ID_INDEX_SCHEMA, 'r') as id_ind_fh:
dynamo_id_ind__cfg = json.load(id_ind_fh)
config.add_dynamo_table_from_json('idIndIndex', names.id_index, **dynamo_id_ind__cfg) # DP XXX
with open(const.DYNAMO_ID_COUNT_SCHEMA, 'r') as id_count_fh:
dynamo_id_count_cfg = json.load(id_count_fh)
config.add_dynamo_table_from_json('idCountIndex', names.id_count_index, **dynamo_id_count_cfg) # DP XXX
return config
def create_config(bosslet_config):
names = bosslet_config.names
session = bosslet_config.session
config = CloudFormationConfiguration('redis', bosslet_config)
vpc_id = config.find_vpc()
internal_subnets, external_subnets = config.find_all_subnets()
sgs = aws.sg_lookup_all(session, vpc_id)
# Create the Cache and CacheState Redis Clusters
REDIS_PARAMETERS = {
"maxmemory-policy": "volatile-lru",
"reserved-memory-percent": str(const.REDIS_RESERVED_MEMORY_PERCENT),
"maxmemory-samples": "5", # ~ 5 - 10
}
config.add_redis_replication("Cache",
names.cache.redis,
internal_subnets, [sgs[names.internal.sg]],
type_=const.REDIS_CACHE_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE,
parameters=REDIS_PARAMETERS)
config.add_redis_replication("CacheState",
names.cache_state.redis,
internal_subnets, [sgs[names.internal.sg]],
type_=const.REDIS_TYPE,
version="3.2.4",
clusters=const.REDIS_CLUSTER_SIZE)
# This one may not be created depending on the scenario type.
if const.REDIS_SESSION_TYPE is not None:
config.add_redis_replication("CacheSession",
names.cache_session.redis,
internal_subnets,
[sgs[names.internal.sg]],
type_=const.REDIS_SESSION_TYPE,
version="3.2.4",
clusters=1)
if const.REDIS_THROTTLE_TYPE is not None:
vpc_id = config.find_vpc()
internal_sg = aws.sg_lookup(session, vpc_id, names.internal.sg)
lambda_subnets, _ = config.find_all_subnets(compatibility='lambda')
config.add_redis_replication("CacheThrottle",
names.cache_throttle.redis,
internal_subnets,
[sgs[names.internal.sg]],
type_=const.REDIS_THROTTLE_TYPE,
version="3.2.4",
clusters=1)
config.add_lambda("CacheThrottleLambda",
names.cache_throttle.lambda_,
aws.role_arn_lookup(session,
'lambda_basic_execution'),
description="Reset Boss throttling metrics",
security_groups=[internal_sg],
subnets=lambda_subnets,
handler='index.handler',
timeout=120,
memory=1024)
# Schedule the lambda to be executed at midnight for the timezone where the bosslet is located
hour = TIMEZONE_OFFSET.get(bosslet_config.REGION, 0)
schedule = 'cron(0 {} * * ? *)'.format(hour)
config.add_cloudwatch_rule(
'CacheThrottleReset',
name=names.cache_throttle.cw,
description='Reset the current Boss throttling metrics',
targets=[
{
'Arn': Arn('CacheThrottleLambda'),
'Id': names.cache_throttle.lambda_,
'Input': json.dumps({'host': names.cache_throttle.redis}),
},
],
schedule=schedule,
depends_on=['CacheThrottleLambda'])
config.add_lambda_permission('CacheThrottlePerms',
names.cache_throttle.lambda_,
principal='events.amazonaws.com',
source=Arn('CacheThrottleReset'))
return config
def create_config(bosslet_config, db_config={}):
names = bosslet_config.names
session = bosslet_config.session
# Lookup IAM Role and SNS Topic ARNs for used later in the config
endpoint_role_arn = aws.role_arn_lookup(session, "endpoint")
cachemanager_role_arn = aws.role_arn_lookup(session, 'cachemanager')
dns_arn = aws.sns_topic_lookup(session, names.dns.sns)
if dns_arn is None:
raise MissingResourceError('SNS topic', names.dns.sns)
mailing_list_arn = aws.sns_topic_lookup(session,
bosslet_config.ALERT_TOPIC)
if mailing_list_arn is None:
raise MissingResourceError('SNS topic', bosslet_config.ALERT_TOPIC)
# Configure Vault and create the user data config that the endpoint will
# use for connecting to Vault and the DB instance
user_data = UserData()
user_data["system"]["fqdn"] = names.endpoint.dns
user_data["system"]["type"] = "endpoint"
user_data["aws"]["db"] = names.endpoint_db.rds
user_data["aws"]["cache"] = names.cache.redis
user_data["aws"]["cache-state"] = names.cache_state.redis
if const.REDIS_SESSION_TYPE is not None:
user_data["aws"]["cache-session"] = names.cache_session.redis
else:
# Don't create a Redis server for dev stacks.
user_data["aws"]["cache-session"] = ''
if const.REDIS_THROTTLE_TYPE is not None:
user_data["aws"]["cache-throttle"] = names.cache_throttle.redis
else:
user_data["aws"]["cache-throttle"] = ''
## cache-db and cache-stat-db need to be in user_data for lambda to access them.
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["cache-throttle-db"] = "0"
user_data["aws"]["cache-session-db"] = "0"
user_data["aws"]["meta-db"] = names.meta.ddb
# Use CloudFormation's Ref function so that queues' URLs are placed into
# the Boss config file.
user_data["aws"]["s3-flush-queue"] = str(Ref(
names.s3flush.sqs)) # str(Ref("S3FlushQueue")) DP XXX
user_data["aws"]["s3-flush-deadletter-queue"] = str(
Ref(names.deadletter.sqs)) #str(Ref("DeadLetterQueue")) DP XXX
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket.s3
user_data["aws"]["tile_bucket"] = names.tile_bucket.s3
user_data["aws"]["ingest_bucket"] = names.ingest_bucket.s3
user_data["aws"]["s3-index-table"] = names.s3_index.ddb
user_data["aws"]["tile-index-table"] = names.tile_index.ddb
user_data["aws"]["id-index-table"] = names.id_index.ddb
user_data["aws"]["id-count-table"] = names.id_count_index.ddb
user_data["aws"]["prod_mailing_list"] = mailing_list_arn
user_data["aws"]["max_task_id_suffix"] = str(const.MAX_TASK_ID_SUFFIX)
user_data["aws"]["id-index-new-chunk-threshold"] = str(
const.DYNAMO_ID_INDEX_NEW_CHUNK_THRESHOLD)
user_data["aws"]["index-deadletter-queue"] = str(
Ref(names.index_deadletter.sqs))
user_data["aws"]["index-cuboids-keys-queue"] = str(
Ref(names.index_cuboids_keys.sqs))
user_data["auth"]["OIDC_VERIFY_SSL"] = str(bosslet_config.VERIFY_SSL)
user_data["lambda"]["flush_function"] = names.multi_lambda.lambda_
user_data["lambda"]["page_in_function"] = names.multi_lambda.lambda_
user_data["lambda"]["ingest_function"] = names.tile_ingest.lambda_
user_data["lambda"]["downsample_volume"] = names.downsample_volume.lambda_
user_data["lambda"]["tile_uploaded_function"] = names.tile_uploaded.lambda_
user_data['sfn']['populate_upload_queue'] = names.ingest_queue_populate.sfn
user_data['sfn']['upload_sfn'] = names.ingest_queue_upload.sfn
user_data['sfn'][
'volumetric_upload_sfn'] = names.volumetric_ingest_queue_upload.sfn
user_data['sfn']['downsample_sfn'] = names.resolution_hierarchy.sfn
user_data['sfn'][
'index_cuboid_supervisor_sfn'] = names.index_cuboid_supervisor.sfn
# Prepare user data for parsing by CloudFormation.
parsed_user_data = {
"Fn::Join": ["", user_data.format_for_cloudformation()]
}
config = CloudFormationConfiguration('api', bosslet_config, version="2")
keypair = bosslet_config.SSH_KEY
vpc_id = config.find_vpc()
internal_subnets, external_subnets = config.find_all_subnets()
az_subnets_asg, external_subnets_asg = config.find_all_subnets(
compatibility='asg')
sgs = aws.sg_lookup_all(session, vpc_id)
# DP XXX: hack until we can get productio updated correctly
config.add_security_group(
'AllHttpHttpsSecurityGroup', names.https.sg,
[('tcp', '443', '443', bosslet_config.HTTPS_INBOUND),
('tcp', '80', '80', bosslet_config.HTTPS_INBOUND)])
sgs[names.https.sg] = Ref('AllHttpHttpsSecurityGroup')
# Create SQS queues and apply access control policies.
# Deadletter queue for indexing operations. This one is populated
# manually by states in the indexing step functions.
config.add_sqs_queue(names.index_deadletter.sqs,
names.index_deadletter.sqs, 30, 20160)
# Queue that holds S3 object keys of cuboids to be indexed.
config.add_sqs_queue(names.index_cuboids_keys.sqs,
names.index_cuboids_keys.sqs, 120, 20160)
#config.add_sqs_queue("DeadLetterQueue", names.deadletter.sqs, 30, 20160) DP XXX
config.add_sqs_queue(names.deadletter.sqs, names.deadletter.sqs, 30, 20160)
max_receives = 3
#config.add_sqs_queue("S3FlushQueue", DP XXX
config.add_sqs_queue(names.s3flush.sqs,
names.s3flush.sqs,
30,
dead=(Arn(names.deadletter.sqs), max_receives))
config.add_sqs_policy(
"sqsEndpointPolicy",
'sqsEndpointPolicy', # DP XXX
[Ref(names.deadletter.sqs),
Ref(names.s3flush.sqs)],
endpoint_role_arn)
config.add_sqs_policy(
"sqsCachemgrPolicy",
'sqsCachemgrPolicy', # DP XXX
[Ref(names.deadletter.sqs),
Ref(names.s3flush.sqs)],
cachemanager_role_arn)
# Create the endpoint ASG, ELB, and RDS instance
cert = aws.cert_arn_lookup(session, names.public_dns("api"))
target_group_keys = config.add_app_loadbalancer(
"EndpointAppLoadBalancer",
names.endpoint_elb.dns, [("443", "80", "HTTPS", cert)],
vpc_id=vpc_id,
subnets=external_subnets_asg,
security_groups=[sgs[names.internal.sg], sgs[names.https.sg]],
public=True)
target_group_arns = [Ref(key) for key in target_group_keys]
config.add_public_dns('EndpointAppLoadBalancer', names.public_dns('api'))
config.add_autoscale_group("Endpoint",
names.endpoint.dns,
aws.ami_lookup(bosslet_config,
names.endpoint.ami),
keypair,
subnets=az_subnets_asg,
type_=const.ENDPOINT_TYPE,
security_groups=[sgs[names.internal.sg]],
user_data=parsed_user_data,
min=const.ENDPOINT_CLUSTER_MIN,
max=const.ENDPOINT_CLUSTER_MAX,
notifications=dns_arn,
role=aws.instance_profile_arn_lookup(
session, 'endpoint'),
health_check_grace_period=90,
detailed_monitoring=True,
target_group_arns=target_group_arns,
depends_on=["EndpointDB"])
# Endpoint servers are not CPU bound typically, so react quickly to load
config.add_autoscale_policy(
"EndpointScaleUp",
Ref("Endpoint"),
adjustments=[
(0.0, 10, 1), # 12% - 22% Utilization add 1 instance
(10, None, 2) # Above 22% Utilization add 2 instances
],
alarms=[("CPUUtilization", "Maximum", "GreaterThanThreshold", "12")],
period=1)
config.add_autoscale_policy(
"EndpointScaleDown",
Ref("Endpoint"),
adjustments=[
(None, 0.0, -1), # Under 1.5% Utilization remove 1 instance
],
alarms=[("CPUUtilization", "Average", "LessThanThreshold", "1.5")],
period=50)
config.add_rds_db("EndpointDB",
names.endpoint_db.dns,
db_config.get("port"),
db_config.get("name"),
db_config.get("user"),
db_config.get("password"),
internal_subnets,
type_=const.RDS_TYPE,
security_groups=[sgs[names.internal.sg]])
# Create the Meta, s3Index, tileIndex, annotation Dynamo tables
with open(const.DYNAMO_METADATA_SCHEMA, 'r') as fh:
dynamo_cfg = json.load(fh)
config.add_dynamo_table_from_json("EndpointMetaDB", names.meta.ddb,
**dynamo_cfg)
with open(const.DYNAMO_S3_INDEX_SCHEMA, 'r') as s3fh:
dynamo_s3_cfg = json.load(s3fh)
config.add_dynamo_table_from_json('s3Index', names.s3_index.ddb,
**dynamo_s3_cfg) # DP XXX
with open(const.DYNAMO_TILE_INDEX_SCHEMA, 'r') as tilefh:
dynamo_tile_cfg = json.load(tilefh)
config.add_dynamo_table_from_json('tileIndex', names.tile_index.ddb,
**dynamo_tile_cfg) # DP XXX
with open(const.DYNAMO_ID_INDEX_SCHEMA, 'r') as id_ind_fh:
dynamo_id_ind__cfg = json.load(id_ind_fh)
config.add_dynamo_table_from_json('idIndIndex', names.id_index.ddb,
**dynamo_id_ind__cfg) # DP XXX
with open(const.DYNAMO_ID_COUNT_SCHEMA, 'r') as id_count_fh:
dynamo_id_count_cfg = json.load(id_count_fh)
config.add_dynamo_table_from_json('idCountIndex', names.id_count_index.ddb,
**dynamo_id_count_cfg) # DP XXX
return config
def create_config(session, domain):
"""Create the CloudFormationConfiguration object."""
config = CloudFormationConfiguration('activities', domain, const.REGION)
names = AWSNames(domain)
global keypair
keypair = aws.keypair_lookup(session)
vpc_id = config.find_vpc(session)
sgs = aws.sg_lookup_all(session, vpc_id)
internal_subnets, _ = config.find_all_availability_zones(session)
internal_subnets_lambda, _ = config.find_all_availability_zones(session, lambda_compatible_only=True)
topic_arn = aws.sns_topic_lookup(session, "ProductionMicronsMailingList")
event_data = {
"lambda-name": "delete_lambda",
"db": names.endpoint_db,
"meta-db": names.meta,
"s3-index-table": names.s3_index,
"id-index-table": names.id_index,
"id-count-table": names.id_count_index,
"cuboid_bucket": names.cuboid_bucket,
"delete_bucket": names.delete_bucket,
"topic-arn": topic_arn,
"query-deletes-sfn-name": names.query_deletes,
"delete-sfn-name": names.delete_cuboid,
"delete-exp-sfn-name": names.delete_experiment,
"delete-coord-frame-sfn-name": names.delete_coord_frame,
"delete-coll-sfn-name": names.delete_collection
}
role_arn = aws.role_arn_lookup(session, "events_for_delete_lambda")
multi_lambda = names.multi_lambda
lambda_arn = aws.lambda_arn_lookup(session, multi_lambda)
target_list = [{
"Arn": lambda_arn,
"Id": multi_lambda,
"Input": json.dumps(event_data)
}]
schedule_expression = "cron(1 6-11/1 ? * TUE-FRI *)"
#schedule_expression = "cron(0/2 * * * ? *)" # testing fire every two minutes
config.add_event_rule("DeleteEventRule", names.delete_event_rule, role_arn=role_arn,
schedule_expression=schedule_expression, target_list=target_list, description=None)
# Events have to be given permission to run lambda.
config.add_lambda_permission('DeleteRulePerm', multi_lambda, principal='events.amazonaws.com',
source=Arn('DeleteEventRule'))
user_data = UserData()
user_data["system"]["fqdn"] = names.activities
user_data["system"]["type"] = "activities"
user_data["aws"]["db"] = names.endpoint_db
user_data["aws"]["cache"] = names.cache
user_data["aws"]["cache-state"] = names.cache_state
user_data["aws"]["cache-db"] = "0"
user_data["aws"]["cache-state-db"] = "0"
user_data["aws"]["meta-db"] = names.meta
user_data["aws"]["cuboid_bucket"] = names.cuboid_bucket
user_data["aws"]["tile_bucket"] = names.tile_bucket
user_data["aws"]["ingest_bucket"] = names.ingest_bucket
user_data["aws"]["s3-index-table"] = names.s3_index
user_data["aws"]["tile-index-table"] = names.tile_index
user_data["aws"]["id-index-table"] = names.id_index
user_data["aws"]["id-count-table"] = names.id_count_index
user_data["aws"]["max_task_id_suffix"] = str(const.MAX_TASK_ID_SUFFIX)
config.add_autoscale_group("Activities",
names.activities,
aws.ami_lookup(session, 'activities.boss'),
keypair,
subnets=internal_subnets_lambda,
type_=const.ACTIVITIES_TYPE,
security_groups=[sgs[names.internal]],
user_data=str(user_data),
role=aws.instance_profile_arn_lookup(session, "activities"),
min=1,
max=1)
config.add_lambda("IngestLambda",
names.ingest_lambda,
aws.role_arn_lookup(session, 'IngestQueueUpload'),
const.INGEST_LAMBDA,
handler="index.handler",
timeout=60 * 5,
memory=3008)
config.add_lambda_permission("IngestLambdaExecute", Ref("IngestLambda"))
# Downsample / Resolution Hierarchy support
lambda_role = aws.role_arn_lookup(session, "lambda_resolution_hierarchy")
config.add_lambda("DownsampleVolumeLambda",
names.downsample_volume_lambda,
lambda_role,
s3=(aws.get_lambda_s3_bucket(session),
"multilambda.{}.zip".format(domain),
"downsample_volume.handler"),
timeout=120,
memory=1024,
runtime='python3.6',
dlq = Ref('DownsampleDLQ'))
config.add_sns_topic("DownsampleDLQ",
names.downsample_dlq,
names.downsample_dlq,
[('lambda', Arn('DownsampleDLQLambda'))])
config.add_lambda('DownsampleDLQLambda',
names.downsample_dlq,
lambda_role,
const.DOWNSAMPLE_DLQ_LAMBDA,
handler='index.handler',
timeout=10)
config.add_lambda_permission('DownsampleDLQLambdaExecute',
Ref('DownsampleDLQLambda'))
return config