def add_parameter_to_family_stack(
self, family, settings: ComposeXSettings,
parameter: Union[str, Parameter]) -> dict:
if self.stack == family.stack:
LOG.warning(
"Cannot add parameter to resource",
f"{self.name}",
"because it is in the same stack as family",
"{family.name}",
)
return self
if (isinstance(parameter, str)
and parameter in self.property_to_parameter_mapping.keys()):
the_parameter = self.property_to_parameter_mapping[parameter]
elif (isinstance(parameter, Parameter)
and parameter in self.property_to_parameter_mapping.values()):
the_parameter = parameter
else:
raise TypeError("parameter must be one of", str, Parameter, "got",
type(parameter))
if self.mappings and self.lookup:
add_update_mapping(
family.template,
self.module.mapping_key,
settings.mappings[self.module.mapping_key],
)
return self.attributes_outputs[the_parameter]
param_id = self.attributes_outputs[the_parameter]
add_parameters(family.template, [param_id["ImportParameter"]])
family.stack.Parameters.update(
{param_id["ImportParameter"].title: param_id["ImportValue"]})
return param_id
def __init__(self, title, settings: ComposeXSettings,
module: XResourceModule, **kwargs):
"""
:param str title: Name of the stack
:param ecs_composex.common.settings.ComposeXSettings settings:
:param dict kwargs:
"""
set_resources(settings, Queue, module)
x_resources = settings.compose_content[module.res_key].values()
lookup_resources = set_lookup_resources(x_resources)
if lookup_resources:
resolve_lookup(lookup_resources, settings, module)
new_resources = set_new_resources(x_resources, True)
if new_resources:
template = build_template(
"SQS template generated by ECS Compose-X")
if lookup_resources:
add_update_mapping(template, module.mapping_key,
settings.mappings[module.mapping_key])
super().__init__(title, stack_template=template, **kwargs)
render_new_queues(settings, new_resources, x_resources, self,
template)
else:
self.is_void = True
for resource in settings.compose_content[module.res_key].values():
resource.stack = self
def handle_import_cognito_pool(the_pool, listener_stack, settings):
"""
Function to map AWS Cognito Pool to attributes
:param the_pool:
:param listener_stack:
:param settings:
:return:
"""
if the_pool.cfn_resource and not the_pool.mappings:
pool_id_param = Parameter(
f"{the_pool.logical_name}{USERPOOL_ID.title}", Type="String")
pool_arn = Parameter(f"{the_pool.logical_name}{USERPOOL_ARN.title}",
Type="String")
add_parameters(listener_stack.stack_template,
[pool_id_param, pool_arn])
listener_stack.Parameters.update({
pool_id_param.title:
Ref(the_pool.cfn_resource),
pool_arn.title:
Ref(pool_arn),
})
return Ref(pool_id_param), Ref(pool_arn)
elif the_pool.mappings and not the_pool.cfn_resource:
add_update_mapping(
listener_stack.stack_template,
the_pool.module.mapping_key,
settings.mappings[the_pool.module.mapping_key],
)
return (
FindInMap(COGNITO_MAP, the_pool.logical_name, USERPOOL_ID.title),
FindInMap(COGNITO_MAP, the_pool.logical_name,
USERPOOL_ARN.return_value),
FindInMap(COGNITO_MAP, the_pool.logical_name,
USERPOOL_DOMAIN.title),
)
def set_from_x_s3(settings, db, db_stack, bucket_name):
"""
Function to link the RDS DB to a Bucket defined in x-s3
:param settings:
:param db:
:param str bucket_name:
:return:
"""
resource = None
if not keyisset(S3_KEY, settings.compose_content):
raise KeyError(
f"No Buckets defined in the Compose file under {S3_KEY}.",
settings.compose_content.keys(),
)
bucket_name = bucket_name.strip("x-s3::")
buckets = settings.compose_content[S3_KEY]
if bucket_name not in [res.name for res in buckets.values()]:
LOG.error(
f"No bucket {bucket_name} in x-s3. Buckets defined: {[res.name for res in buckets.values()]}"
)
return
for resource in buckets.values():
if bucket_name == resource.name:
break
if not resource:
return
if resource.cfn_resource:
return get_s3_bucket_arn_from_resource(db_stack, resource)
elif resource.lookup and keyisset("s3", settings.mappings):
add_update_mapping(db_stack.stack_template, "s3",
settings.mappings["s3"])
return resource.lookup_properties[S3_BUCKET_ARN]
def initialize_family_services(settings: ComposeXSettings,
family: ComposeFamily) -> None:
"""
Function to handle creation of services within the same family.
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
if settings.secrets_mappings:
add_update_mapping(family.template, SECRETS_KEY,
settings.secrets_mappings)
add_update_mapping(
family.iam_manager.exec_role.stack.stack_template,
SECRETS_KEY,
settings.secrets_mappings,
)
family.init_task_definition()
family.set_secrets_access()
family.ecs_service = EcsService(family)
family.stack.Parameters.update({
ecs_params.SERVICE_NAME_T: family.logical_name,
CLUSTER_NAME_T: Ref(CLUSTER_NAME),
ROOT_STACK_NAME_T: Ref(ROOT_STACK_NAME),
})
upload_services_env_files(family, settings)
set_repository_credentials(family, settings)
set_volumes(family)
family.handle_alarms()
def import_resource_into_service_stack(
settings: ComposeXSettings,
resource,
family: ComposeFamily,
params_to_add,
params_values,
) -> None:
"""
Function to either add parameters to the services stack or mapping for a given resource
:param ecs_composex.common.settings.ComposeXSettings settings:
:param ecs_composex.common.compose_resources.ServicesXResource resource: The resource
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
:param list[ecs_composex.common.cfn_params.Parameter] params_to_add:
:param dict params_values:
"""
if resource.cfn_resource:
add_parameters(family.template, params_to_add)
family.stack.Parameters.update(params_values)
elif resource.mappings:
add_update_mapping(
family.template,
resource.module.mapping_key,
settings.mappings[resource.module.mapping_key],
)
def define_vpc_settings(settings: ComposeXSettings,
vpc_module: XResourceModule, vpc_stack: ComposeXStack):
"""
Function to deal with vpc stack settings
"""
if settings.requires_vpc() and not vpc_stack.vpc_resource:
LOG.info(
f"{settings.name} - Services or x-Resources need a VPC to function. Creating default one"
)
vpc_stack.create_new_default_vpc("vpc", vpc_module, settings)
settings.root_stack.stack_template.add_resource(vpc_stack)
vpc_stack.vpc_resource.generate_outputs()
elif (vpc_stack.is_void and vpc_stack.vpc_resource
and vpc_stack.vpc_resource.mappings):
vpc_stack.vpc_resource.generate_outputs()
add_update_mapping(
settings.root_stack.stack_template,
"Network",
vpc_stack.vpc_resource.mappings,
)
elif (vpc_stack.vpc_resource and vpc_stack.vpc_resource.cfn_resource
and vpc_stack.title
not in settings.root_stack.stack_template.resources.keys()):
settings.root_stack.stack_template.add_resource(vpc_stack)
LOG.info(
f"{settings.name}.x-vpc - VPC stack added. A new VPC will be created."
)
vpc_stack.vpc_resource.generate_outputs()
def assign_kms_key_to_queue(kms_key, queue, queue_stack, settings):
"""
Assigns the KMS Key pointer to the queue property
:param ecs_composex.kms.kms_stack.KmsKey kms_key:
:param queue:
:param ecs_composex.sqs.sqs_stack.XStack queue_stack:
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
kms_key_id = kms_key.attributes_outputs[KMS_KEY_ID]
if kms_key.cfn_resource:
add_parameters(queue_stack.stack_template,
[kms_key_id["ImportParameter"]])
setattr(
queue.cfn_resource,
KEY,
Ref(kms_key_id["ImportParameter"]),
)
queue_stack.Parameters.update(
{kms_key_id["ImportParameter"].title: kms_key_id["ImportValue"]})
elif not kms_key.cfn_resource and kms_key.mappings:
add_update_mapping(
queue.stack.stack_template,
kms_key.module.mapping_key,
settings.mappings[kms_key.module.mapping_key],
)
setattr(queue.cfn_resource, KEY, kms_key_id["ImportValue"])
def s3_to_firehose(
resource: Bucket,
dest_resource: DeliveryStream,
dest_resource_stack,
settings: ComposeXSettings,
) -> None:
"""
Updates
:param Bucket resource:
:param DeliveryStream dest_resource:
:param dest_resource_stack:
:param settings:
:return:
"""
if not dest_resource.cfn_resource:
LOG.error(
f"{dest_resource.module.res_key}.{dest_resource.name} - Not a new resource"
)
for prop_path, bucket_param in FIREHOSE_PROPERTIES.items():
prop_attr = get_dest_resource_nested_property(
prop_path, dest_resource.cfn_resource)
if skip_if(resource, prop_attr):
continue
bucket_id = resource.attributes_outputs[bucket_param]
if resource.cfn_resource:
add_parameters(dest_resource_stack.stack_template,
[bucket_id["ImportParameter"]])
setattr(
prop_attr[0],
prop_attr[1],
Ref(bucket_id["ImportParameter"]),
)
dest_resource.stack.Parameters.update(
{bucket_id["ImportParameter"].title: bucket_id["ImportValue"]})
arn_pointer = Ref(bucket_id["ImportParameter"])
elif not resource.cfn_resource and resource.mappings:
add_update_mapping(
dest_resource.stack.stack_template,
resource.module.mapping_key,
settings.mappings[resource.module.mapping_key],
)
setattr(prop_attr[0], prop_attr[1], bucket_id["ImportValue"])
arn_pointer = bucket_id["ImportValue"]
else:
raise ValueError(
resource.module.mapping_key,
resource.name,
"Unable to determine if new or lookup",
)
map_x_resource_perms_to_resource(
dest_resource,
arn_value=arn_pointer,
access_definition="s3destination",
access_subkey="kinesis_firehose",
resource_policies=get_access_types(resource.module.mod_key),
resource_mapping_key=resource.module.mapping_key,
)
dest_resource.ensure_iam_policies_dependencies()
def set_all_mappings_to_root_stack(root_stack: ComposeXStack,
settings: ComposeXSettings):
"""
Adds all of the mappings to the root stack
:param ComposeXStack root_stack:
:param ecs_composex.common.settings.ComposeXSettings settings: The settings for the execution
"""
for mapping_key, mapping in settings.mappings.items():
add_update_mapping(root_stack.stack_template, mapping_key, mapping)
def import_dbs(db: NetworkXResource | DatabaseXResource,
settings: ComposeXSettings) -> None:
"""
Function to go over each service defined in the DB and assign found DB settings to service
"""
for target in db.families_targets:
add_update_mapping(
target[0].template,
db.module.mapping_key,
settings.mappings[db.module.mapping_key],
)
handle_import_dbs_to_services(db, target)
def update_stack_with_resource_settings(
property_stack: ComposeXStack,
the_resource: Certificate,
the_property,
property_name: str,
settings: ComposeXSettings,
) -> None:
"""
Assigns the CFN pointer to the value to replace.
If it is a new certificate, it will add the parameter to get the cert ARN and set the parameter stack value
If it is a Lookup certificate, it will add the mapping to the stack and set FindInMap to the certificate ARN
:param ComposeXStack property_stack:
:param Certificate the_resource:
:param the_property:
:param property_name:
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
if the_resource.cfn_resource:
add_parameters(
property_stack.stack_template,
[the_resource.attributes_outputs[CERT_ARN]["ImportParameter"]],
)
property_stack.Parameters.update(
{
the_resource.attributes_outputs[CERT_ARN][
"ImportParameter"
].title: the_resource.attributes_outputs[CERT_ARN]["ImportValue"]
}
)
setattr(
the_property,
property_name,
Ref(the_resource.attributes_outputs[CERT_ARN]["ImportParameter"]),
)
elif the_resource.mappings:
if keyisset(
the_resource.module.mapping_key, property_stack.stack_template.mappings
):
property_stack.stack_template.mappings[the_resource.module.mapping_key][
the_resource.logical_name
].update(the_resource.mappings)
else:
add_update_mapping(
property_stack.stack_template,
the_resource.module.mapping_key,
settings.mappings[the_resource.module.mapping_key],
)
setattr(
the_property,
property_name,
the_resource.attributes_outputs[CERT_ARN]["ImportValue"],
)
def process_return_values(
namespace: PrivateNamespace,
resource,
return_values: dict,
instance_props: dict,
settings,
):
"""
Processes the ReturnValues attributes to assign to an instance.
:param namespace:
:param ecs_composex.compose.x_resources.XResource resource:
:param dict return_values:
:param dict instance_props:
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
for key, value in return_values.items():
for attribute_param in resource.attributes_outputs.keys():
if attribute_param.title == key or (
attribute_param.return_value and attribute_param.return_value == key
):
break
else:
raise KeyError(
f"{resource.module.res_key}.{resource.name}"
" - ReturnValue {key} not found. Available",
[p.title for p in resource.attributes_outputs.keys()],
)
attribute_pointer = resource.attributes_outputs[attribute_param]
if resource.cfn_resource:
add_parameters(
namespace.stack.stack_template, [attribute_pointer["ImportParameter"]]
)
namespace.stack.Parameters.update(
{
attribute_pointer["ImportParameter"].title: attribute_pointer[
"ImportValue"
]
}
)
instance_props["InstanceAttributes"][key] = Ref(
attribute_pointer["ImportParameter"]
)
else:
add_update_mapping(
namespace.stack.stack_template,
resource.module.mapping_key,
settings.mappings[resource.module.mapping_key],
)
instance_props["InstanceAttributes"][value] = attribute_pointer[
"ImportValue"
]
def set_from_definition(self, root_stack, session, settings):
if self.lookup:
self.lookup_cluster(session)
add_update_mapping(root_stack.stack_template, self.mappings_key,
self.mappings)
elif self.definition and self.use:
self.mappings = {CLUSTER_NAME.title: {"Name": self.use}}
add_update_mapping(root_stack.stack_template, self.mappings_key,
self.mappings)
self.cluster_identifier = FindInMap(self.mappings_key,
CLUSTER_NAME.title, "Name")
elif self.properties:
self.define_cluster(root_stack, settings)
def set_for_lookup_kms_key(prop_attr, resource, resource_id, dest_resource,
settings) -> AWSHelperFn:
add_update_mapping(
dest_resource.stack.stack_template,
resource.module.mapping_key,
settings.mappings[resource.module.mapping_key],
)
setattr(prop_attr[0], prop_attr[1], resource_id["ImportValue"])
if resource.is_cmk:
setattr(prop_attr[0], "KeyType", "CUSTOMER_MANAGED_CMK")
else:
setattr(prop_attr[0], "KeyType", "AWS_OWNED_CMK")
return resource_id["ImportValue"]
def process_dns_config(
namespace: PrivateNamespace,
resource: NetworkXResource | DatabaseXResource,
dns_settings: dict,
settings: ComposeXSettings,
service: Service,
instance: Instance,
) -> None:
"""
Process the DnsSettings of the x-cloudmap configuration
"""
hostname = (
dns_settings["Hostname"]
if keyisset("Hostname", dns_settings)
else resource.logical_name
)
if namespace.zone_name not in hostname:
hostname = f"{hostname}.{namespace.zone_name}"
record = DnsRecord(Type="CNAME", TTL="15")
config = DnsConfig(DnsRecords=[record], NamespaceId=NoValue)
setattr(service, "DnsConfig", config)
setattr(service, "Name", hostname)
if not hasattr(resource, "db_cluster_endpoint_param"):
return
attribute_pointer = resource.attributes_outputs[resource.db_cluster_endpoint_param]
if resource.cfn_resource:
add_parameters(
namespace.stack.stack_template, [attribute_pointer["ImportParameter"]]
)
namespace.stack.Parameters.update(
{
attribute_pointer["ImportParameter"].title: attribute_pointer[
"ImportValue"
]
}
)
instance.InstanceAttributes.update(
{"AWS_INSTANCE_CNAME": Ref(attribute_pointer["ImportParameter"])}
)
else:
add_update_mapping(
namespace.stack.stack_template,
resource.module.mapping_key,
settings.mappings[resource.module.mapping_key],
)
instance.InstanceAttributes.update(
{"AWS_INSTANCE_CNAME": attribute_pointer["ImportValue"]}
)
def init_stack_for_resources(self, settings) -> None:
"""
When creating new Route53 records, if the x-route53 where looked up, we need to initialize the Route53 stack
:param ComposeXStack root_stack: The root stack
"""
if self.stack.is_void:
stack_template = build_template("Root stack for x-cloudmap resources")
super(XStack, self.stack).__init__(MOD_KEY, stack_template)
self.stack.is_void = False
add_update_mapping(
self.stack.stack_template,
self.module.mapping_key,
settings.mappings[self.module.mapping_key],
)
def lookup_dns_zone(route53_zone, validation_option, acm_stack, settings):
"""
Update the HostedZoneId property when using a lookup DNS zone
:param route53_zone:
:param troposphere.certificatemanager.DomainValidationOption validation_option:
:param XStack acm_stack:
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
add_update_mapping(
acm_stack.stack_template,
route53_zone.module.mapping_key,
settings.mappings[route53_zone.module.mapping_key],
)
zone_id_attribute = route53_zone.attributes_outputs[PUBLIC_DNS_ZONE_ID]
setattr(validation_option, "HostedZoneId", zone_id_attribute["ImportValue"])
def handle_x_dependencies(self, settings, root_stack) -> None:
"""
WIll go over all the new resources to create in the execution and search for properties that can be updated
with itself
:param ecs_composex.common.settings.ComposeXSettings settings:
:param ComposeXStack root_stack: The root stack
"""
for resource in settings.get_x_resources(include_mappings=False):
if not resource.cfn_resource:
continue
resource_stack = resource.stack
if not resource_stack:
LOG.error(
f"resource {resource.name} has no `stack` attribute defined. Skipping"
)
continue
mappings = [
(Elbv2, handle_elbv2_records),
(Certificate, handle_acm_records),
]
for target in mappings:
if isinstance(resource, target[0]) or issubclass(
type(resource), target[0]
):
if (
self.mappings
and self.stack
and not self.stack.is_void
and self.stack.stack_template
):
add_update_mapping(
self.stack.stack_template,
self.module.mapping_key,
settings.mappings[self.module.mapping_key],
)
target[1](
self, self.stack, resource, resource_stack, settings, root_stack
)
def handle_compose_topics(alarm, alarms_stack, settings, topic_def, notify_on):
"""
Function to handle x-alarms to x-sns topics
:param ecs_composex.alarms.alarms_stack.Alarm alarm:
:param ecs_composex.common.stacks.ComposeXStack alarms_stack:
:param ecs_composex.common.settings.ComposeXSettings settings:
:param dict topic_def:
:param str notify_on:
:return:
"""
try:
topic = settings.compose_content[SNS_KEY][topic_def[SNS_KEY]]
except KeyError:
LOG.error(
f"No topic {topic_def[SNS_KEY]} found in {SNS_KEY}.{Topic.keyword}"
)
raise
if not topic.attributes_outputs:
topic.init_outputs()
topic.generate_outputs()
topic_arn = topic.attributes_outputs[TOPIC_ARN]
if topic.cfn_resource:
add_parameters(alarms_stack.stack_template,
[topic_arn["ImportParameter"]])
alarms_stack.Parameters.update(
{topic_arn["Name"]: topic_arn["ImportValue"]})
map_topic_to_action(alarm, notify_on,
Ref(topic_arn["ImportParameter"]))
else:
add_update_mapping(
alarms_stack.stack_template,
topic.module.mapping_key,
settings.mappings[topic.module.mapping_key],
)
map_topic_to_action(alarm, notify_on, topic_arn["ImportValue"])
def create_registry(family, namespace, port_config, settings):
"""
Creates the settings for the ECS Service Registries and adds the resources to the appropriate template
:param ecs_composex.ecs.ecs_family.ComposeFamily family:
:param ecs_composex.cloudmap.cloudmap_stack.PrivateNamespace namespace:
:param dict port_config:
:param ecs_composex.common.settings.ComposeXSettings settings:
"""
if family.ecs_service.registries:
LOG.warn(f"{family.name} already has a CloudMap mapping. "
f"Only one can be set. Ignoring mapping to {namespace.name}")
return
if namespace.cfn_resource:
add_parameters(
family.template,
[
namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]
["ImportParameter"]
],
)
family.stack.Parameters.update({
namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]["ImportParameter"].title:
namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]["ImportValue"]
})
namespace_id = Ref(namespace.attributes_outputs[PRIVATE_NAMESPACE_ID]
["ImportParameter"])
elif namespace.lookup_properties:
add_update_mapping(
family.template,
namespace.module.mapping_key,
settings.mappings[namespace.module.mapping_key],
)
namespace_id = namespace.attributes_outputs[PRIVATE_NAMESPACE_ID][
"ImportValue"]
else:
raise AttributeError(
f"{namespace.module.res_key}.{namespace.name} - Cannot define if new or lookup !?"
)
sd_service = SdService(
f"{namespace.logical_name}EcsServiceDiscovery{family.logical_name}",
Description=Sub(f"{family.name} service"),
NamespaceId=namespace_id,
HealthCheckCustomConfig=HealthCheckCustomConfig(FailureThreshold=1.0),
DnsConfig=DnsConfig(
RoutingPolicy="MULTIVALUE",
NamespaceId=Ref(AWS_NO_VALUE),
DnsRecords=[
DnsRecord(TTL="15", Type="A"),
DnsRecord(TTL="15", Type="SRV"),
],
),
Name=family.family_hostname,
)
service_registry = ServiceRegistry(
f"ServiceRegistry{port_config['target']}",
RegistryArn=GetAtt(sd_service, "Arn"),
Port=int(port_config["target"]),
)
add_resource(family.template, sd_service)
family.ecs_service.registries.append(service_registry)