def create_kms_template(settings):
"""
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
mono_template = False
template = build_template("Root template for KMS")
if not keyisset(RES_KEY, settings.compose_content):
return
keys = settings.compose_content[RES_KEY]
if len(list(keys.keys())) <= CFN_MAX_OUTPUTS:
mono_template = True
for key_name in keys:
key_res_name = NONALPHANUM.sub("", key_name)
key = generate_key(key_name, key_res_name, keys[key_name])
if key:
values = [
(KMS_KEY_ARN_T, "Arn", GetAtt(key, "Arn")),
(KMS_KEY_ID_T, "Name", Ref(key)),
]
outputs = ComposeXOutput(key, values, True)
if mono_template:
template.add_resource(key)
handle_key_settings(template, key, keys[key_name])
template.add_output(outputs.outputs)
elif not mono_template:
key_template = build_template(f"Template for DynamoDB key {key.title}")
key_template.add_resource(key)
key_template.add_output(outputs.outputs)
key_stack = ComposeXStack(key_res_name, stack_template=key_template)
template.add_resource(key_stack)
return template
def generate_sqs_root_template(settings):
"""
Function to create the root DynamdoDB template.
:param ecs_composex.common.settings.ComposeXSettings settings: Execution settings.
:return:
"""
mono_template = False
output_per_resource = 2
if not keyisset(RES_KEY, settings.compose_content):
return None
queues = settings.compose_content[RES_KEY]
if (len(list(queues.keys())) * output_per_resource) <= CFN_MAX_OUTPUTS:
mono_template = True
template = build_template("DynamoDB for ECS ComposeX")
for queue_name in queues:
queue_res_name = NONALPHANUM.sub("", queue_name)
queue_def = queues[queue_name]
queue = define_queue(queue_name, queue_def, queues, mono_template)
if queue:
values = [
(SQS_URL, "Url", Ref(queue)),
(SQS_ARN_T, "Arn", GetAtt(queue, "Arn")),
(SQS_NAME_T, "Name", Ref(queue)),
]
outputs = ComposeXOutput(queue,
values,
duplicate_attr=(not mono_template))
if mono_template:
template.add_resource(queue)
template.add_output(outputs.outputs)
elif not mono_template:
parameters = {}
if hasattr(queue, "RedrivePolicy"):
parameters.update({
DLQ_ARN_T:
GetAtt(
NONALPHANUM.sub(
"",
queue_def["Properties"]["RedrivePolicy"]
["deadLetterTargetArn"],
),
f"Outputs.{SQS_ARN_T}",
)
})
queue_template = build_template(
f"Template for SQS queue {queue.title}", [DLQ_ARN])
queue_template.add_resource(queue)
queue_template.add_output(outputs.outputs)
queue_stack = ComposeXStack(
queue_res_name,
stack_template=queue_template,
stack_parameters=parameters,
)
template.add_resource(queue_stack)
return template
def generate_sns_templates(settings):
"""
Entrypoint function to generate the SNS topics templates
:param settings:
:type settings: ecs_composex.common.settings.ComposeXSettings
:return:
"""
allowed_keys = [TOPICS_KEY, SUBSCRIPTIONS_KEY]
res_content = settings.compose_content[RES_KEY]
if not set(res_content).issubset(allowed_keys):
raise KeyError(
"SNS Only supports two types of resources",
allowed_keys,
"provided",
res_content.keys(),
)
root_template = build_template("SNS Root Template")
res_count = define_resources(res_content)
if keyisset(TOPICS_KEY, res_content):
add_sns_topics(root_template, settings.compose_content, res_count)
if keyisset(SUBSCRIPTIONS_KEY, res_content):
pass
add_topics_outputs(root_template)
return root_template
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
"""
Method to initialize the XStack for Events
:param str title: title for the stack
:param ecs_composex.common.settings.ComposeXSettings settings: Execution settings
:param dict kwargs:
"""
set_resources(settings, Rule, module)
x_resources = settings.compose_content[module.res_key].values()
lookup_resources = set_lookup_resources(x_resources)
if lookup_resources:
warnings.warn(
f"{module.res_key} does not support Lookup/Use. You can only create new resources"
)
new_resources = set_new_resources(x_resources, False)
if new_resources:
stack_template = build_template(
"Events rules for ComposeX",
[CLUSTER_NAME, FARGATE_VERSION],
)
super().__init__(title, stack_template, **kwargs)
create_events_template(self, settings, new_resources)
else:
self.is_void = True
def create_root_template(new_resources: list, module_res_key: str) -> Template:
"""
Function to create the root stack template for profiles
:param list[CodeProfiler] new_resources:
:param str module_res_key:
:return: The template wit the profiles
:rtype: troposphere.Template
"""
root_tpl = build_template(f"Root stack to manage {module_res_key}")
for res in new_resources:
try:
props = import_record_properties(res.properties,
ProfilingGroup,
ignore_missing_required=False)
if res.parameters and keyisset("AppendStackId", res.parameters):
props["ProfilingGroupName"] = Sub(
f"{res.properties['ProfilingGroupName']}-${{StackId}}",
StackId=STACK_ID_SHORT,
)
except KeyError:
props = import_record_properties(res.properties,
ProfilingGroup,
ignore_missing_required=True)
props["ProfilingGroupName"] = Sub(
f"{res.logical_name}-${{StackId}}", StackId=STACK_ID_SHORT)
res.cfn_resource = ProfilingGroup(res.logical_name, **props)
res.init_outputs()
res.generate_outputs()
add_outputs(root_tpl, res.outputs)
root_tpl.add_resource(res.cfn_resource)
return root_tpl
def generate_compute_template(settings):
"""
Function that generates the Compute resources to run ECS services on top of EC2
:param ComposeXSettings settings: The settings for executio
:return: ECS Cluster Template
:rtype: troposphere.Template
"""
template = build_template(
"Cluster template generated by ECS Compose X",
[
USE_FLEET,
USE_ONDEMAND,
compute_params.ECS_AMI_ID,
compute_params.TARGET_CAPACITY,
compute_params.MIN_CAPACITY,
compute_params.MAX_CAPACITY,
vpc_params.APP_SUBNETS,
vpc_params.VPC_ID,
CLUSTER_NAME,
],
)
template.add_condition(
compute_conditions.MAX_IS_MIN_T, compute_conditions.MAX_IS_MIN
)
template.add_condition(cfn_conditions.USE_SPOT_CON_T, cfn_conditions.USE_SPOT_CON)
launch_template = add_hosts_resources(template)
add_spotfleet_stack(template, settings, launch_template)
return template
def create_s3_template(new_buckets: list[Bucket],
template: Template) -> Template:
"""
Function to create the root S3 template.
:param list new_buckets:
:param troposphere.Template template:
:return:
"""
mono_template = False
if len(list(new_buckets)) <= COMPOSEX_MAX_OUTPUTS:
mono_template = True
for bucket in new_buckets:
generate_bucket(bucket)
if bucket.cfn_resource:
bucket.init_outputs()
bucket.generate_outputs()
bucket_template = template
if mono_template:
bucket_template.add_resource(bucket.cfn_resource)
bucket_template.add_output(bucket.outputs)
elif not mono_template:
bucket_template = build_template(
f"Template for S3 Bucket {bucket.name}")
bucket_template.add_resource(bucket.cfn_resource)
bucket_template.add_output(bucket.outputs)
bucket_stack = ComposeXStack(bucket.logical_name,
stack_template=bucket_template)
template.add_resource(bucket_stack)
evaluate_parameters(bucket, bucket_template)
return template
def create_kms_template(template, new_keys, xstack):
"""
Function to create all the KMS Keys based on their definition
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
mono_template = False
if len(new_keys) <= CFN_MAX_OUTPUTS:
mono_template = True
for key in new_keys:
key.stack = xstack
key.define_kms_key()
if key and key.cfn_resource:
key.init_outputs()
key.generate_outputs()
if mono_template:
template.add_resource(key.cfn_resource)
key.handle_key_settings(template)
template.add_output(key.outputs)
elif not mono_template:
key_template = build_template(
f"Template for KMS key {key.logical_name}")
key_template.add_resource(key.cfn_resource)
key.handle_key_settings(key_template)
key_template.add_output(key.outputs)
key_stack = ComposeXStack(key.logical_name,
stack_template=key_template)
template.add_resource(key_stack)
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
set_resources(settings, NeptuneDBCluster, module)
x_resources = settings.compose_content[module.res_key].values()
new_resources = set_new_resources(x_resources, True)
lookup_resources = set_lookup_resources(x_resources)
if new_resources:
stack_template = build_template(
"Root template for Neptune by ComposeX",
[VPC_ID, STORAGE_SUBNETS])
super().__init__(title, stack_template, **kwargs)
create_neptune_template(stack_template, new_resources, settings,
self)
else:
self.is_void = True
if lookup_resources:
if not keyisset(module.mapping_key, settings.mappings):
settings.mappings[module.mapping_key] = {}
for resource in lookup_resources:
resource.lookup_resource(
NEPTUNE_DB_CLUSTER_ARN_RE,
get_db_cluster_config,
CfnDBCluster.resource_type,
"rds:cluster",
)
resource.generate_cfn_mappings_from_lookup_properties()
resource.generate_outputs()
settings.mappings[module.mapping_key].update(
{resource.logical_name: resource.mappings})
for resource in settings.compose_content[module.res_key].values():
resource.stack = self
def __init__(self, title: str, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
"""
:param str title:
:param ecs_composex.common.settings.ComposeXSettings settings:
:param dict kwargs:
"""
set_resources(settings, UserPool, module)
x_resources = settings.compose_content[module.res_key].values()
lookup_resources = set_lookup_resources(x_resources)
if lookup_resources:
resolve_lookup(lookup_resources, settings, module)
new_resources = set_new_resources(x_resources, False)
if new_resources:
LOG.error(
f"{module.res_key} does not support new resources creation yet."
)
stack_template = build_template(
f"Root stack to manage {module.mod_key}")
super().__init__(title, stack_template, **kwargs)
self.is_void = True
else:
self.is_void = True
for resource in x_resources:
resource.stack = self
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
"""
:param str title: Name of the stack
:param ecs_composex.common.settings.ComposeXSettings settings:
:param dict kwargs:
"""
set_resources(settings, Queue, module)
x_resources = settings.compose_content[module.res_key].values()
lookup_resources = set_lookup_resources(x_resources)
if lookup_resources:
resolve_lookup(lookup_resources, settings, module)
new_resources = set_new_resources(x_resources, True)
if new_resources:
template = build_template(
"SQS template generated by ECS Compose-X")
if lookup_resources:
add_update_mapping(template, module.mapping_key,
settings.mappings[module.mapping_key])
super().__init__(title, stack_template=template, **kwargs)
render_new_queues(settings, new_resources, x_resources, self,
template)
else:
self.is_void = True
for resource in settings.compose_content[module.res_key].values():
resource.stack = self
def create_dynamodb_template(new_tables, template, self_stack):
"""
Function to create the root DynamdoDB template.
:param list new_tables:
:param troposhere.Template template:
:param ComposeXStack self_stack:
:return:
"""
total_outputs = sum(
len(table.attributes_outputs.keys()) for table in new_tables)
mono_template = False
if total_outputs <= CFN_MAX_OUTPUTS:
mono_template = True
for table in new_tables:
if mono_template:
table.stack = self_stack
define_table(table, template)
elif not mono_template:
table_template = build_template(
f"Template for DynamoDB table {table.title}")
table_stack = ComposeXStack(table.logical_name,
stack_template=table_template)
table.stack = table_stack
define_table(table, table_template)
template.add_resource(table_stack)
return template
def create_efs_stack(settings, new_resources):
"""
Function to create the root stack and add EFS FS.
:param ecs_composex.common.settings.ComposeXSettings settings:
:param list new_resources:
:return: Root template for EFS
:rtype: troposphere.Template
"""
template = build_template("Root for EFS built by ECS Compose-X", [FS_PORT])
for res in new_resources:
res_cfn_props = import_record_properties(res.properties, FileSystem)
res.cfn_resource = FileSystem(res.logical_name, **res_cfn_props)
res.db_sg = SecurityGroup(
f"{res.logical_name}SecurityGroup",
GroupName=Sub(f"{res.logical_name}EfsSg"),
GroupDescription=Sub(f"SG for EFS {res.cfn_resource.title}"),
VpcId=Ref(VPC_ID),
)
template.add_resource(res.cfn_resource)
template.add_resource(res.db_sg)
res.init_outputs()
res.generate_outputs()
template.add_output(res.outputs)
return template
def init_elbv2_template():
"""
Function to create a new root ELBv2 stack
:return:
"""
lb_params = [VPC_ID, APP_SUBNETS, PUBLIC_SUBNETS]
template = build_template("elbv2 root template for ComposeX", lb_params)
return template
def init_doc_db_template():
"""
Function to generate the base of the DocDB template.
:return: the root template
:rtype: troposphere.Template
"""
template = build_template("Root template for DocumentDB for ComposeX",
[VPC_ID, STORAGE_SUBNETS])
return template
def init_vpc_template() -> troposphere.Template:
"""
Simple wrapper function to create the VPC Template
:rtype: troposhere.Template
"""
template = build_template("Vpc Template generated via ECS Compose-X", )
template.add_mapping("AwsLbAccounts", aws_mappings.AWS_LB_ACCOUNTS)
return template
def init_rds_root_template():
"""
Function to generate the root template for RDS
:return: template
:rtype: troposphere.Template
"""
template = build_template("RDS Root Template", [VPC_ID, STORAGE_SUBNETS])
return template
def init_database_template(db):
"""
Function to initialize the DB Template
:param db: The DB definition
:return: template
:rtype: troposphere.Template
"""
template = build_template(
f"Template for RDS DB {db.name}",
[
VPC_ID,
DB_ENGINE_NAME,
DB_ENGINE_VERSION,
DB_NAME,
DB_USERNAME,
DB_SNAPSHOT_ID,
DB_PASSWORD_LENGTH,
DB_INSTANCE_CLASS,
DB_STORAGE_CAPACITY,
DB_STORAGE_TYPE,
STORAGE_SUBNETS,
],
)
template.add_condition(
rds_conditions.USE_DB_SNAPSHOT_CON_T, rds_conditions.USE_DB_SNAPSHOT_CON
)
template.add_condition(
rds_conditions.NOT_USE_DB_SNAPSHOT_CON_T,
rds_conditions.NOT_USE_DB_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.USE_CLUSTER_CON_T, rds_conditions.USE_CLUSTER_CON
)
template.add_condition(
rds_conditions.NOT_USE_CLUSTER_CON_T, rds_conditions.NOT_USE_CLUSTER_CON
)
template.add_condition(
rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON_T,
rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON_T,
rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON_T,
rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON_T,
rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON,
)
return template
def test_zone_create(content, zone_create):
"""
Tests zone lookup
:return:
"""
print(content)
updated_content = content.copy()
updated_content.update(zone_create)
settings = create_settings(updated_content, "x_dns")
root_stack = ComposeXStack("root", build_template())
DnsSettings(root_stack, settings, Ref("VpcId"))
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
params = {
CLUSTER_NAME.title: settings.ecs_cluster,
}
stack_template = build_template("Root template for Dashboards",
[CLUSTER_NAME])
super().__init__(title,
stack_template,
stack_parameters=params,
**kwargs)
create_dashboards(settings, self, module)
def init_database_template(db_name):
"""
Function to initialize the DB Template
:param str db_name: Name of the DB as defined in compose file
:return: template
:rtype: troposphere.Template
"""
template = build_template(
f"Template for RDS DB {db_name}",
[
VPC_ID,
DB_ENGINE_NAME,
DB_ENGINE_VERSION,
STORAGE_SUBNETS,
DBS_SUBNET_GROUP,
DB_NAME,
DB_USERNAME,
DB_SNAPSHOT_ID,
DB_PASSWORD_LENGTH,
DB_INSTANCE_CLASS,
DB_STORAGE_CAPACITY,
DB_STORAGE_TYPE,
],
)
template.add_condition(rds_conditions.DBS_SUBNET_GROUP_CON_T,
rds_conditions.DBS_SUBNET_GROUP_CON)
template.add_condition(rds_conditions.USE_DB_SNAPSHOT_CON_T,
rds_conditions.USE_DB_SNAPSHOT_CON)
template.add_condition(rds_conditions.NOT_USE_DB_SNAPSHOT_CON_T,
rds_conditions.NOT_USE_DB_SNAPSHOT_CON)
template.add_condition(rds_conditions.USE_CLUSTER_CON_T,
rds_conditions.USE_CLUSTER_CON)
template.add_condition(rds_conditions.NOT_USE_CLUSTER_CON_T,
rds_conditions.NOT_USE_CLUSTER_CON)
template.add_condition(
rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON_T,
rds_conditions.USE_CLUSTER_AND_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON_T,
rds_conditions.USE_CLUSTER_NOT_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON_T,
rds_conditions.NOT_USE_CLUSTER_USE_SNAPSHOT_CON,
)
template.add_condition(
rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON_T,
rds_conditions.USE_CLUSTER_OR_SNAPSHOT_CON,
)
create_db_subnet_group(template, True)
return template
def init_stack_for_records(self, root_stack) -> None:
"""
When creating new Route53 records, if the x-route53 where looked up, we need to initialize the Route53 stack
:param ComposeXStack root_stack: The root stack
"""
if self.module.mapping_key not in root_stack.stack_template.resources:
from ecs_composex.route53.route53_stack import XStack
stack_template = build_template(self.stack.stack_title)
super(XStack, self.stack).__init__(self.module.mapping_key, stack_template)
self.stack.is_void = False
root_stack.stack_template.add_resource(self.stack)
def test_zone_lookup(content, zone_lookup):
"""
Tests zone lookup
:return:
"""
updated_content = content.copy()
updated_content.update(zone_lookup)
settings = create_settings(updated_content, "x_dns")
root_stack = ComposeXStack("root", build_template())
try:
DnsSettings(root_stack, settings, Ref("VpcId"))
except Exception:
pass
def init_root_template():
"""
Function to initialize the root template
:return: template
:rtype: troposphere.Template
"""
template = build_template(
"Root template generated via ECS ComposeX",
[USE_FLEET, USE_ONDEMAND, CLUSTER_NAME],
)
return template
def create_dynamodb_template(settings):
"""
Function to create the root DynamdoDB template.
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
mono_template = False
if not keyisset(RES_KEY, settings.compose_content):
return None
tables = settings.compose_content[RES_KEY]
if len(list(tables.keys())) <= CFN_MAX_OUTPUTS:
mono_template = True
template = build_template("DynamoDB for ECS ComposeX")
for table_name in tables:
table_res_name = NONALPHANUM.sub("", table_name)
table = generate_table(table_name, table_res_name, tables[table_name])
if table:
values = [
(TABLE_ARN_T, "Arn", GetAtt(table, "Arn")),
(TABLE_NAME_T, "Name", Ref(table)),
]
outputs = ComposeXOutput(table, values, True)
if mono_template:
template.add_resource(table)
template.add_output(outputs.outputs)
elif not mono_template:
table_template = build_template(
f"Template for DynamoDB table {table.title}"
)
table_template.add_resource(table)
table_template.add_output(outputs.outputs)
table_stack = ComposeXStack(
table_res_name, stack_template=table_template
)
template.add_resource(table_stack)
return template
def create_streams_template(new_resources, settings):
"""
Function to create the root template for Kinesis streams
:param list<ecs_composex.kinesis.kinesis_stack.Stream> new_resources:
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
root_template = build_template("Root stack for ecs_composex.kinesis")
for res in new_resources:
create_new_stream(res)
root_template.add_resource(res.cfn_resource)
root_template.add_output(res.outputs)
return root_template
def create_acm_template(settings):
"""
Main entrypoint for ACM root template creation
:param ecs_composex.common.settings.ComposeXSettings settings: The execution settings
:return: root stack template for ACM.
:rtype: troposphere.Template
"""
if not keyisset(RES_KEY, settings.compose_content):
return
certs = settings.compose_content[RES_KEY]
root_acm_tpl = build_template("Root template for ACM")
add_certificates(root_acm_tpl, certs)
return root_acm_tpl
def __init__(
self, title, settings: ComposeXSettings, module: XResourceModule, **kwargs
):
set_resources(settings, Rds, module)
x_resources = settings.compose_content[module.res_key].values()
new_resources = set_new_resources(x_resources, True)
lookup_resources = set_lookup_resources(x_resources)
if new_resources:
stack_template = build_template(
"Root stack for RDS DBs", [VPC_ID, STORAGE_SUBNETS]
)
super().__init__(title, stack_template, **kwargs)
generate_rds_templates(stack_template, new_resources, settings)
self.mark_nested_stacks()
else:
self.is_void = True
for resource in settings.compose_content[module.res_key].values():
resource.stack = self
if lookup_resources and module.mapping_key not in settings.mappings:
settings.mappings[module.mapping_key] = {}
for resource in lookup_resources:
if keyisset("cluster", resource.lookup):
resource.lookup_resource(
RDS_DB_CLUSTER_ARN_RE,
get_db_cluster_config,
CfnDBCluster.resource_type,
"rds:cluster",
"cluster",
)
elif keyisset("db", resource.lookup):
resource.lookup_resource(
RDS_DB_INSTANCE_ARN_RE,
get_db_instance_config,
CfnDBInstance.resource_type,
"rds:db",
"db",
)
else:
raise KeyError(
f"{resource.module.res_key}.{resource.name} - "
"You must specify the cluster or instance to lookup"
)
if keyisset("secret", resource.lookup):
lookup_rds_secret(resource, resource.lookup["secret"])
resource.generate_cfn_mappings_from_lookup_properties()
resource.generate_outputs()
settings.mappings[module.mapping_key].update(
{resource.logical_name: resource.mappings}
)
def __init__(self, name: str, settings: ComposeXSettings, **kwargs):
stack_template = build_template("Root stack for IAM Roles")
add_parameters(stack_template, [CLUSTER_NAME])
super().__init__(name, stack_template, **kwargs)
exec_role_managed_policy = add_ecs_execution_role_managed_policy(stack_template)
self.Parameters.update(
{CLUSTER_NAME.title: settings.ecs_cluster.cluster_identifier}
)
new_roles = import_family_roles(settings, exec_role_managed_policy)
for role in new_roles:
self.stack_template.add_resource(role.cfn_resource)
role.stack = self
if not role.attributes_outputs:
role.generate_outputs()
add_outputs(stack_template, role.outputs)
def init_stack_for_resources(self, settings) -> None:
"""
When creating new Route53 records, if the x-route53 where looked up, we need to initialize the Route53 stack
:param ComposeXStack root_stack: The root stack
"""
if self.stack.is_void:
stack_template = build_template("Root stack for x-cloudmap resources")
super(XStack, self.stack).__init__(MOD_KEY, stack_template)
self.stack.is_void = False
add_update_mapping(
self.stack.stack_template,
self.module.mapping_key,
settings.mappings[self.module.mapping_key],
)
