def test_lookup(existing_cluster, nonexisting_cluster):
"""
Function to test the dynamodb table lookup
"""
here = path.abspath(path.dirname(__file__))
session = boto3.session.Session()
pill = placebo.attach(session, data_path=f"{here}/x_ecs")
pill.playback()
template = Template()
stack = ComposeXStack("test", stack_template=template)
settings = ComposeXSettings(
content=existing_cluster,
session=session,
**{
ComposeXSettings.name_arg: "test",
ComposeXSettings.command_arg: ComposeXSettings.render_arg,
ComposeXSettings.format_arg: "yaml",
},
)
cluster = add_ecs_cluster(settings, stack)
assert cluster is False
template = Template()
stack = ComposeXStack("test", stack_template=template)
settings = ComposeXSettings(
content=existing_cluster,
session=session,
**{
ComposeXSettings.name_arg: "test",
ComposeXSettings.command_arg: ComposeXSettings.render_arg,
ComposeXSettings.format_arg: "yaml",
},
)
cluster = add_ecs_cluster(settings, stack)
assert cluster is True
def define_vpc_settings(settings: ComposeXSettings,
vpc_module: XResourceModule, vpc_stack: ComposeXStack):
"""
Function to deal with vpc stack settings
"""
if settings.requires_vpc() and not vpc_stack.vpc_resource:
LOG.info(
f"{settings.name} - Services or x-Resources need a VPC to function. Creating default one"
)
vpc_stack.create_new_default_vpc("vpc", vpc_module, settings)
settings.root_stack.stack_template.add_resource(vpc_stack)
vpc_stack.vpc_resource.generate_outputs()
elif (vpc_stack.is_void and vpc_stack.vpc_resource
and vpc_stack.vpc_resource.mappings):
vpc_stack.vpc_resource.generate_outputs()
add_update_mapping(
settings.root_stack.stack_template,
"Network",
vpc_stack.vpc_resource.mappings,
)
elif (vpc_stack.vpc_resource and vpc_stack.vpc_resource.cfn_resource
and vpc_stack.title
not in settings.root_stack.stack_template.resources.keys()):
settings.root_stack.stack_template.add_resource(vpc_stack)
LOG.info(
f"{settings.name}.x-vpc - VPC stack added. A new VPC will be created."
)
vpc_stack.vpc_resource.generate_outputs()
def add_spotfleet_stack(template, settings, launch_template):
"""
Function to build the spotfleet stack and add it to the Cluster parent template
:param launch_template: the launch template
:param troposphere.Template template: parent cluster template
:param ComposeXSettings settings: The settings for execution
"""
compute_config = ComputeConfig(settings)
parameters = {
ROOT_STACK_NAME_T: If(
cfn_conditions.USE_STACK_NAME_CON_T,
Ref("AWS::StackName"),
Ref(ROOT_STACK_NAME),
),
compute_params.LAUNCH_TEMPLATE_ID_T: Ref(launch_template),
compute_params.LAUNCH_TEMPLATE_VersionNumber_T: GetAtt(
launch_template, "LatestVersionNumber"
),
compute_params.MAX_CAPACITY_T: Ref(compute_params.MAX_CAPACITY),
compute_params.MIN_CAPACITY_T: Ref(compute_params.MIN_CAPACITY),
compute_params.TARGET_CAPACITY_T: Ref(compute_params.TARGET_CAPACITY),
}
fleet_template = generate_spot_fleet_template(settings, compute_config.spot_config)
template.add_resource(
ComposeXStack(
"SpotFleet",
stack_template=fleet_template,
Condition=cfn_conditions.USE_SPOT_CON_T,
Parameters=parameters,
)
)
def create_s3_template(new_buckets: list[Bucket],
template: Template) -> Template:
"""
Function to create the root S3 template.
:param list new_buckets:
:param troposphere.Template template:
:return:
"""
mono_template = False
if len(list(new_buckets)) <= COMPOSEX_MAX_OUTPUTS:
mono_template = True
for bucket in new_buckets:
generate_bucket(bucket)
if bucket.cfn_resource:
bucket.init_outputs()
bucket.generate_outputs()
bucket_template = template
if mono_template:
bucket_template.add_resource(bucket.cfn_resource)
bucket_template.add_output(bucket.outputs)
elif not mono_template:
bucket_template = build_template(
f"Template for S3 Bucket {bucket.name}")
bucket_template.add_resource(bucket.cfn_resource)
bucket_template.add_output(bucket.outputs)
bucket_stack = ComposeXStack(bucket.logical_name,
stack_template=bucket_template)
template.add_resource(bucket_stack)
evaluate_parameters(bucket, bucket_template)
return template
def create_kms_template(template, new_keys, xstack):
"""
Function to create all the KMS Keys based on their definition
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
mono_template = False
if len(new_keys) <= CFN_MAX_OUTPUTS:
mono_template = True
for key in new_keys:
key.stack = xstack
key.define_kms_key()
if key and key.cfn_resource:
key.init_outputs()
key.generate_outputs()
if mono_template:
template.add_resource(key.cfn_resource)
key.handle_key_settings(template)
template.add_output(key.outputs)
elif not mono_template:
key_template = build_template(
f"Template for KMS key {key.logical_name}")
key_template.add_resource(key.cfn_resource)
key.handle_key_settings(key_template)
key_template.add_output(key.outputs)
key_stack = ComposeXStack(key.logical_name,
stack_template=key_template)
template.add_resource(key_stack)
def create_kms_template(settings):
"""
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
mono_template = False
template = build_template("Root template for KMS")
if not keyisset(RES_KEY, settings.compose_content):
return
keys = settings.compose_content[RES_KEY]
if len(list(keys.keys())) <= CFN_MAX_OUTPUTS:
mono_template = True
for key_name in keys:
key_res_name = NONALPHANUM.sub("", key_name)
key = generate_key(key_name, key_res_name, keys[key_name])
if key:
values = [
(KMS_KEY_ARN_T, "Arn", GetAtt(key, "Arn")),
(KMS_KEY_ID_T, "Name", Ref(key)),
]
outputs = ComposeXOutput(key, values, True)
if mono_template:
template.add_resource(key)
handle_key_settings(template, key, keys[key_name])
template.add_output(outputs.outputs)
elif not mono_template:
key_template = build_template(f"Template for DynamoDB key {key.title}")
key_template.add_resource(key)
key_template.add_output(outputs.outputs)
key_stack = ComposeXStack(key_res_name, stack_template=key_template)
template.add_resource(key_stack)
return template
def create_dynamodb_template(new_tables, template, self_stack):
"""
Function to create the root DynamdoDB template.
:param list new_tables:
:param troposhere.Template template:
:param ComposeXStack self_stack:
:return:
"""
total_outputs = sum(
len(table.attributes_outputs.keys()) for table in new_tables)
mono_template = False
if total_outputs <= CFN_MAX_OUTPUTS:
mono_template = True
for table in new_tables:
if mono_template:
table.stack = self_stack
define_table(table, template)
elif not mono_template:
table_template = build_template(
f"Template for DynamoDB table {table.title}")
table_stack = ComposeXStack(table.logical_name,
stack_template=table_template)
table.stack = table_stack
define_table(table, table_template)
template.add_resource(table_stack)
return template
def generate_sqs_root_template(settings):
"""
Function to create the root DynamdoDB template.
:param ecs_composex.common.settings.ComposeXSettings settings: Execution settings.
:return:
"""
mono_template = False
output_per_resource = 2
if not keyisset(RES_KEY, settings.compose_content):
return None
queues = settings.compose_content[RES_KEY]
if (len(list(queues.keys())) * output_per_resource) <= CFN_MAX_OUTPUTS:
mono_template = True
template = build_template("DynamoDB for ECS ComposeX")
for queue_name in queues:
queue_res_name = NONALPHANUM.sub("", queue_name)
queue_def = queues[queue_name]
queue = define_queue(queue_name, queue_def, queues, mono_template)
if queue:
values = [
(SQS_URL, "Url", Ref(queue)),
(SQS_ARN_T, "Arn", GetAtt(queue, "Arn")),
(SQS_NAME_T, "Name", Ref(queue)),
]
outputs = ComposeXOutput(queue,
values,
duplicate_attr=(not mono_template))
if mono_template:
template.add_resource(queue)
template.add_output(outputs.outputs)
elif not mono_template:
parameters = {}
if hasattr(queue, "RedrivePolicy"):
parameters.update({
DLQ_ARN_T:
GetAtt(
NONALPHANUM.sub(
"",
queue_def["Properties"]["RedrivePolicy"]
["deadLetterTargetArn"],
),
f"Outputs.{SQS_ARN_T}",
)
})
queue_template = build_template(
f"Template for SQS queue {queue.title}", [DLQ_ARN])
queue_template.add_resource(queue)
queue_template.add_output(outputs.outputs)
queue_stack = ComposeXStack(
queue_res_name,
stack_template=queue_template,
stack_parameters=parameters,
)
template.add_resource(queue_stack)
return template
def test_zone_create(content, zone_create):
"""
Tests zone lookup
:return:
"""
print(content)
updated_content = content.copy()
updated_content.update(zone_create)
settings = create_settings(updated_content, "x_dns")
root_stack = ComposeXStack("root", build_template())
DnsSettings(root_stack, settings, Ref("VpcId"))
def test_zone_lookup(content, zone_lookup):
"""
Tests zone lookup
:return:
"""
updated_content = content.copy()
updated_content.update(zone_lookup)
settings = create_settings(updated_content, "x_dns")
root_stack = ComposeXStack("root", build_template())
try:
DnsSettings(root_stack, settings, Ref("VpcId"))
except Exception:
pass
def render_new_queues(settings, new_queues, queues, xstack, template):
"""
Function to create the root DynamdoDB template.
:param ecs_composex.common.settings.ComposeXSettings settings: Execution settings.
"""
mono_template = False
output_per_resource = 3
if (len(new_queues) * output_per_resource) <= CFN_MAX_OUTPUTS:
mono_template = True
for queue in new_queues:
queue.stack = xstack
define_queue(queue, queues, mono_template)
if queue.cfn_resource:
queue.init_outputs()
queue.generate_outputs()
if mono_template:
template.add_resource(queue.cfn_resource)
template.add_output(queue.outputs)
elif not mono_template:
parameters = {}
if hasattr(queue, "RedrivePolicy"):
parameters.update({
DLQ_ARN_T:
GetAtt(
NONALPHANUM.sub(
"",
queues[queue.name].definition["Properties"]
["RedrivePolicy"]["deadLetterTargetArn"],
),
f"Outputs.{SQS_ARN_T}",
)
})
queue_template = build_template(
f"Template for SQS queue {queue.cfn_resource.title}",
[DLQ_ARN],
)
queue_template.add_resource(queue.cfn_resource)
queue_template.add_output(queue.outputs)
queue_stack = ComposeXStack(
queue.logical_name,
stack_template=queue_template,
stack_parameters=parameters,
)
template.add_resource(queue_stack)
def create_root_stack(settings: ComposeXSettings) -> ComposeXStack:
"""
Initializes the root stack template and ComposeXStack
:param ecs_composex.common.settings.ComposeXSettings settings: The settings for the execution
"""
template = init_template("Root template generated via ECS ComposeX")
template.add_mapping("ComposeXDefaults",
{"ECS": {
"PlatformVersion": "1.4.0"
}})
root_stack_title = NONALPHANUM.sub("", settings.name.title())
root_stack = ComposeXStack(
root_stack_title,
stack_template=template,
file_name=settings.name,
)
return root_stack
def add_certificates(acm_tpl, certs):
"""
Function to add all the ACM certs together
:param acm_tpl:
:param certs:
:return:
"""
for cert_name in certs:
resource_name = NONALPHANUM.sub("", cert_name)
cert_def = certs[cert_name]
cert_props = cert_def["Properties"]
cert_params = build_cert_params(cert_props)
cert_params.update(pass_root_stack_name())
cert_template = initialize_acm_stack_template(resource_name)
acm_tpl.add_resource(
ComposeXStack(
resource_name,
stack_template=cert_template,
Parameters=cert_params,
))
def generate_full_template(settings):
"""
Function to generate the root root_template
:param ecs_composex.common.settings.ComposeXSettings settings: The settings for the execution
:return root_template: Template, params
:rtype: root_template, list
"""
LOG.debug(settings)
root_stack = ComposeXStack(settings.name,
stack_template=init_root_template())
dns_inputs(root_stack)
vpc_stack = add_vpc_to_root(root_stack, settings)
dns_settings = DnsSettings(root_stack, settings, get_vpc_id(vpc_stack))
root_stack.Parameters.update(dns_settings.root_params)
create_cluster = add_ecs_cluster(settings, root_stack)
compute_stack = add_compute(root_stack.stack_template, settings, vpc_stack)
if create_cluster and settings.create_compute and compute_stack:
compute_stack.DependsOn.append(ROOT_CLUSTER_NAME)
services_families = define_services_families(
settings.compose_content[SERVICES_KEY])
services_stack = create_services(root_stack, settings, vpc_stack,
dns_settings.nested_params,
create_cluster)
add_x_resources(root_stack.stack_template, settings, vpc_stack=vpc_stack)
apply_x_configs_to_ecs(settings, root_stack.stack_template, services_stack,
services_families)
apply_x_to_x_configs(root_stack.stack_template, settings)
if keyisset("x-appmesh", settings.compose_content):
mesh = Mesh(
settings.compose_content["x-appmesh"],
services_families,
services_stack,
settings,
)
mesh.render_mesh_template(services_stack)
add_all_tags(root_stack.stack_template, settings)
return root_stack
def create_dynamodb_template(settings):
"""
Function to create the root DynamdoDB template.
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
mono_template = False
if not keyisset(RES_KEY, settings.compose_content):
return None
tables = settings.compose_content[RES_KEY]
if len(list(tables.keys())) <= CFN_MAX_OUTPUTS:
mono_template = True
template = build_template("DynamoDB for ECS ComposeX")
for table_name in tables:
table_res_name = NONALPHANUM.sub("", table_name)
table = generate_table(table_name, table_res_name, tables[table_name])
if table:
values = [
(TABLE_ARN_T, "Arn", GetAtt(table, "Arn")),
(TABLE_NAME_T, "Name", Ref(table)),
]
outputs = ComposeXOutput(table, values, True)
if mono_template:
template.add_resource(table)
template.add_output(outputs.outputs)
elif not mono_template:
table_template = build_template(
f"Template for DynamoDB table {table.title}"
)
table_template.add_resource(table)
table_template.add_output(outputs.outputs)
table_stack = ComposeXStack(
table_res_name, stack_template=table_template
)
template.add_resource(table_stack)
return template
def add_sns_topics(root_template, content, res_count, count=170):
"""
Function to add SNS topics to the root template
:param int count: quantity of resources that should trigger the split into nested stacks
:param root_template:
:param content:
:param int res_count: Number of resources created related to SNS
:param kwargs:
:return:
"""
if res_count > count:
LOG.info(
f"There are more than {count} resources to handle for SNS. Splitting into nested stacks"
)
template = build_template("Root stack for SNS topics")
add_topics_to_template(template, content[RES_KEY][TOPICS_KEY], content)
add_topics_outputs(template)
root_template.add_resource(
ComposeXStack(title=TOPICS_STACK_NAME, stack_template=template)
)
else:
add_topics_to_template(root_template, content[RES_KEY][TOPICS_KEY], content)
def add_db_stack(root_template, dbs_subnet_group, db_name, db, settings):
"""
Function to add the DB stack to the root stack
:param dbs_subnet_group: Subnet group for DBs
:type dbs_subnet_group: troposphere.rds.DBSubnetGroup
:param root_template: root template to add the nested stack to
:type root_template: troposphere.Template
:param db_name: name of the DB as defined in the x-rds section
:type db_name: str
:param db: the database definition from the compose file
:type db: dict
:param kwargs: extra arguments
"""
props = db["Properties"]
required_props = [DB_ENGINE_NAME_T, DB_ENGINE_VERSION_T]
validate_kwargs(required_props, props)
non_stack_params = {
DB_ENGINE_NAME_T: props[DB_ENGINE_NAME_T],
DB_ENGINE_VERSION_T: props[DB_ENGINE_VERSION_T],
}
parameters = {
VPC_ID_T: Ref(VPC_ID),
DBS_SUBNET_GROUP_T: Ref(dbs_subnet_group),
DB_NAME_T: db_name,
STORAGE_SUBNETS_T: Join(",", Ref(STORAGE_SUBNETS)),
ROOT_STACK_NAME_T: Ref(ROOT_STACK_NAME),
}
parameters.update(non_stack_params)
db_template = generate_database_template(db_name, db)
if db_template is None:
return
db_stack = ComposeXStack(
db_name, stack_template=db_template, stack_parameters=parameters
)
root_template.add_resource(db_stack)
def root_stack():
tpl = Template("Template for testing")
return ComposeXStack("root", stack_template=tpl)